From 9499353f851212976ef0e34f34200b54a154d697 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Thu, 28 May 2026 16:08:16 -0700 Subject: [PATCH 01/38] operator: add extension Context type --- pkg/operator/context.go | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 pkg/operator/context.go diff --git a/pkg/operator/context.go b/pkg/operator/context.go new file mode 100644 index 0000000000..cb84bbd052 --- /dev/null +++ b/pkg/operator/context.go @@ -0,0 +1,41 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package operator + +import ( + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/tls/certificatemanagement" +) + +// Context carries reconcile-derived inputs from controllers into render +// modifiers. OSS code never reads these fields - only registered modifiers do. +// Two kinds of value live here: +// - raw cluster state gathered generically (Installation, FelixConfiguration, +// ClusterDomain) that modifiers derive their own values from, and +// - controller-produced artifacts (TrustedBundle, NodePrometheusTLS) that can +// only be created controller-side because they have cluster side effects. +type Context struct { + Installation *operatorv1.InstallationSpec + FelixConfiguration *v3.FelixConfiguration + ClusterDomain string + + // TrustedBundle is the shared CA bundle for the calico-system namespace. + TrustedBundle certificatemanagement.TrustedBundle + + // NodePrometheusTLS is produced by the installation controller's enterprise + // extension and mounted by the node modifier. + NodePrometheusTLS certificatemanagement.KeyPairInterface +} From a5767f2913c0db85c3ca059cbf6657dc47a1dbf8 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Thu, 28 May 2026 16:11:25 -0700 Subject: [PATCH 02/38] operator: add patch registry --- pkg/operator/operator_suite_test.go | 27 +++++++++++++++ pkg/operator/patch.go | 54 +++++++++++++++++++++++++++++ pkg/operator/patch_test.go | 54 +++++++++++++++++++++++++++++ 3 files changed, 135 insertions(+) create mode 100644 pkg/operator/operator_suite_test.go create mode 100644 pkg/operator/patch.go create mode 100644 pkg/operator/patch_test.go diff --git a/pkg/operator/operator_suite_test.go b/pkg/operator/operator_suite_test.go new file mode 100644 index 0000000000..42d3e32f63 --- /dev/null +++ b/pkg/operator/operator_suite_test.go @@ -0,0 +1,27 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package operator_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestOperator(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "pkg/operator Suite") +} diff --git a/pkg/operator/patch.go b/pkg/operator/patch.go new file mode 100644 index 0000000000..29d84dbc09 --- /dev/null +++ b/pkg/operator/patch.go @@ -0,0 +1,54 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package operator + +import "sigs.k8s.io/controller-runtime/pkg/client" + +// PatchFunc post-processes the objects a render component produced. It may +// mutate matched objects and/or append additional objects, and must return the +// (possibly extended) slice. Implementations self-gate on ctx.Installation.Variant. +type PatchFunc func(ctx Context, objs []client.Object) []client.Object + +var patches = map[string][]PatchFunc{} + +// Patch registers fn to run against the named component's objects. Multiple +// patches may be registered for the same component; they run in registration order. +func Patch(component string, fn PatchFunc) { + patches[component] = append(patches[component], fn) +} + +// ApplyPatches runs every patch registered for the named component over objs. +func ApplyPatches(component string, ctx Context, objs []client.Object) []client.Object { + for _, fn := range patches[component] { + objs = fn(ctx, objs) + } + return objs +} + +// FindObject returns the first object of type T with the given name. +func FindObject[T client.Object](objs []client.Object, name string) (T, bool) { + var zero T + for _, o := range objs { + if t, ok := o.(T); ok && o.GetName() == name { + return t, true + } + } + return zero, false +} + +// ResetForTest clears all registries. Test-only. +func ResetForTest() { + patches = map[string][]PatchFunc{} +} diff --git a/pkg/operator/patch_test.go b/pkg/operator/patch_test.go new file mode 100644 index 0000000000..d9cd4de108 --- /dev/null +++ b/pkg/operator/patch_test.go @@ -0,0 +1,54 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package operator_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/tigera/operator/pkg/operator" +) + +var _ = Describe("patch registry", func() { + AfterEach(func() { + operator.ResetForTest() + }) + + It("applies a registered patch to the matching component", func() { + operator.Patch("test", func(ctx operator.Context, objs []client.Object) []client.Object { + cm, ok := operator.FindObject[*corev1.ConfigMap](objs, "cm") + Expect(ok).To(BeTrue()) + cm.Data = map[string]string{"k": "v"} + return append(objs, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "extra"}}) + }) + + in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} + out := operator.ApplyPatches("test", operator.Context{}, in) + + Expect(out).To(HaveLen(2)) + cm := out[0].(*corev1.ConfigMap) + Expect(cm.Data).To(HaveKeyWithValue("k", "v")) + Expect(out[1].GetName()).To(Equal("extra")) + }) + + It("returns objects unchanged when no patch is registered", func() { + in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} + out := operator.ApplyPatches("unregistered", operator.Context{}, in) + Expect(out).To(Equal(in)) + }) +}) From b4482276533440d16711c7b5d85127367170aab1 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Thu, 28 May 2026 16:13:59 -0700 Subject: [PATCH 03/38] operator: add image-override registry --- pkg/operator/image.go | 44 +++++++++++++++++++++++++++++++++++ pkg/operator/image_test.go | 47 ++++++++++++++++++++++++++++++++++++++ pkg/operator/patch.go | 1 + 3 files changed, 92 insertions(+) create mode 100644 pkg/operator/image.go create mode 100644 pkg/operator/image_test.go diff --git a/pkg/operator/image.go b/pkg/operator/image.go new file mode 100644 index 0000000000..b253611254 --- /dev/null +++ b/pkg/operator/image.go @@ -0,0 +1,44 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package operator + +import ( + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/components" +) + +// ImageOverride returns the component image to use for an installation, and +// false to decline (leaving the default in place). Implementations self-gate +// on in.Variant. +type ImageOverride func(in *operatorv1.InstallationSpec) (components.Component, bool) + +var imageOverrides = map[string]ImageOverride{} + +// OverrideImage registers an image override under key. The key is the render +// component's image identifier (e.g. "node"). +func OverrideImage(key string, fn ImageOverride) { + imageOverrides[key] = fn +} + +// ResolveImage returns the override registered for key if it applies to in, +// otherwise def. Render components call this inside ResolveImages. +func ResolveImage(key string, def components.Component, in *operatorv1.InstallationSpec) components.Component { + if fn, ok := imageOverrides[key]; ok { + if c, override := fn(in); override { + return c + } + } + return def +} diff --git a/pkg/operator/image_test.go b/pkg/operator/image_test.go new file mode 100644 index 0000000000..93e9beb67d --- /dev/null +++ b/pkg/operator/image_test.go @@ -0,0 +1,47 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package operator_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/operator" +) + +var _ = Describe("image overrides", func() { + AfterEach(func() { + operator.ResetForTest() + }) + + It("uses the override when one matches", func() { + operator.OverrideImage("node", func(in *operatorv1.InstallationSpec) (components.Component, bool) { + if !in.Variant.IsEnterprise() { + return components.Component{}, false + } + return components.ComponentTigeraNode, true + }) + + ent := &operatorv1.InstallationSpec{Variant: operatorv1.TigeraSecureEnterprise} + Expect(operator.ResolveImage("node", components.ComponentCalicoNode, ent)).To(Equal(components.ComponentTigeraNode)) + }) + + It("falls back to the default when no override matches", func() { + oss := &operatorv1.InstallationSpec{Variant: operatorv1.Calico} + Expect(operator.ResolveImage("node", components.ComponentCalicoNode, oss)).To(Equal(components.ComponentCalicoNode)) + }) +}) diff --git a/pkg/operator/patch.go b/pkg/operator/patch.go index 29d84dbc09..94721a76bd 100644 --- a/pkg/operator/patch.go +++ b/pkg/operator/patch.go @@ -51,4 +51,5 @@ func FindObject[T client.Object](objs []client.Object, name string) (T, bool) { // ResetForTest clears all registries. Test-only. func ResetForTest() { patches = map[string][]PatchFunc{} + imageOverrides = map[string]ImageOverride{} } From e651d7db4479239becea914109c2c06be7a9eb61 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Thu, 28 May 2026 16:14:55 -0700 Subject: [PATCH 04/38] render: add Named interface and component name constants for extension keying --- pkg/render/component.go | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/pkg/render/component.go b/pkg/render/component.go index eea911fbe3..585355438d 100644 --- a/pkg/render/component.go +++ b/pkg/render/component.go @@ -1,4 +1,4 @@ -// Copyright (c) 2021-2024 Tigera, Inc. All rights reserved. +// Copyright (c) 2021-2026 Tigera, Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -39,3 +39,18 @@ type Component interface { // that create pods. Return OSTypeAny means that no node selector should be set for the "kubernetes.io/os" label. SupportedOSType() rmeta.OSType } + +// Named is implemented by components that expose enterprise extension points. +// The componentHandler uses Name() to look up registered patches. Components +// without enterprise extensions need not implement it. +type Named interface { + Name() string +} + +// Component names used as keys into the operator patch registry. Keep these in +// sync with the Name() methods that return them. +const ( + ComponentNameTypha = "typha" + ComponentNameNode = "node" + ComponentNameAPIServer = "apiserver" +) From c1cd817a8f1d16cbe2573dd416afa2ee5d82b64f Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Thu, 28 May 2026 16:42:07 -0700 Subject: [PATCH 05/38] utils: apply registered render patches in componentHandler Add WithContext/ComponentHandlerOption to NewComponentHandler (variadic, backward-compatible) and call operator.ApplyPatches in CreateOrUpdateOrDelete for components implementing render.Named. --- .../gatewayapi/gatewayapi_controller.go | 2 +- .../gatewayapi/gatewayapi_controller_test.go | 2 +- .../installation/core_controller.go | 2 +- .../installation/core_controller_test.go | 18 ++++---- pkg/controller/utils/component.go | 21 ++++++++- pkg/controller/utils/component_test.go | 43 +++++++++++++++++++ 6 files changed, 74 insertions(+), 14 deletions(-) diff --git a/pkg/controller/gatewayapi/gatewayapi_controller.go b/pkg/controller/gatewayapi/gatewayapi_controller.go index cdbcd38237..d949c8c959 100644 --- a/pkg/controller/gatewayapi/gatewayapi_controller.go +++ b/pkg/controller/gatewayapi/gatewayapi_controller.go @@ -147,7 +147,7 @@ type ReconcileGatewayAPI struct { status status.StatusManager clusterDomain string multiTenant bool - newComponentHandler func(log logr.Logger, client client.Client, scheme *runtime.Scheme, cr metav1.Object) utils.ComponentHandler + newComponentHandler func(log logr.Logger, client client.Client, scheme *runtime.Scheme, cr metav1.Object, opts ...utils.ComponentHandlerOption) utils.ComponentHandler watchEnvoyProxy func(namespacedName operatorv1.NamespacedName) error watchEnvoyGateway func(namespacedName operatorv1.NamespacedName) error } diff --git a/pkg/controller/gatewayapi/gatewayapi_controller_test.go b/pkg/controller/gatewayapi/gatewayapi_controller_test.go index cf918699a0..9a968e880a 100644 --- a/pkg/controller/gatewayapi/gatewayapi_controller_test.go +++ b/pkg/controller/gatewayapi/gatewayapi_controller_test.go @@ -702,7 +702,7 @@ var _ = Describe("Gateway API controller tests", func() { var fakeComponentHandlers []*fakeComponentHandler -func FakeComponentHandler(log logr.Logger, client client.Client, scheme *runtime.Scheme, cr metav1.Object) utils.ComponentHandler { +func FakeComponentHandler(log logr.Logger, client client.Client, scheme *runtime.Scheme, cr metav1.Object, opts ...utils.ComponentHandlerOption) utils.ComponentHandler { h := &fakeComponentHandler{ client: client, scheme: scheme, diff --git a/pkg/controller/installation/core_controller.go b/pkg/controller/installation/core_controller.go index a018c16364..89acae7416 100644 --- a/pkg/controller/installation/core_controller.go +++ b/pkg/controller/installation/core_controller.go @@ -405,7 +405,7 @@ type ReconcileInstallation struct { kubernetesVersion *common.VersionInfo // newComponentHandler returns a new component handler. Useful stub for unit testing. - newComponentHandler func(log logr.Logger, client client.Client, scheme *runtime.Scheme, cr metav1.Object) utils.ComponentHandler + newComponentHandler func(log logr.Logger, client client.Client, scheme *runtime.Scheme, cr metav1.Object, opts ...utils.ComponentHandlerOption) utils.ComponentHandler } // GetActivePools returns the full set of enabled IP pools in the cluster. diff --git a/pkg/controller/installation/core_controller_test.go b/pkg/controller/installation/core_controller_test.go index aa113b34e3..5676d0ebe9 100644 --- a/pkg/controller/installation/core_controller_test.go +++ b/pkg/controller/installation/core_controller_test.go @@ -2458,7 +2458,7 @@ var _ = Describe("Testing core-controller installation", func() { migrationChecked: true, tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } @@ -2605,7 +2605,7 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { manageCRDs: true, v3CRDs: true, kubernetesVersion: &common.VersionInfo{Major: 1, Minor: 32}, - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } @@ -2638,7 +2638,7 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { manageCRDs: true, v3CRDs: true, kubernetesVersion: &common.VersionInfo{Major: 1, Minor: 31}, - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } @@ -2657,7 +2657,7 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { manageCRDs: true, v3CRDs: false, kubernetesVersion: &common.VersionInfo{Major: 1, Minor: 32}, - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } @@ -2675,7 +2675,7 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { manageCRDs: false, v3CRDs: true, kubernetesVersion: &common.VersionInfo{Major: 1, Minor: 32}, - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } @@ -2693,7 +2693,7 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { manageCRDs: true, v3CRDs: true, kubernetesVersion: nil, - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } @@ -2728,7 +2728,7 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { manageCRDs: true, v3CRDs: true, kubernetesVersion: &common.VersionInfo{Major: 1, Minor: 32}, - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } @@ -2787,7 +2787,7 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { manageCRDs: true, v3CRDs: true, kubernetesVersion: &common.VersionInfo{Major: 1, Minor: 32}, - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } @@ -2810,7 +2810,7 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { manageCRDs: true, v3CRDs: true, kubernetesVersion: &common.VersionInfo{Major: 1, Minor: 32}, - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } diff --git a/pkg/controller/utils/component.go b/pkg/controller/utils/component.go index b6679a2647..292aa99479 100644 --- a/pkg/controller/utils/component.go +++ b/pkg/controller/utils/component.go @@ -46,6 +46,7 @@ import ( "github.com/tigera/operator/pkg/apigroup" "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/controller/status" + "github.com/tigera/operator/pkg/operator" "github.com/tigera/operator/pkg/render" rmeta "github.com/tigera/operator/pkg/render/common/meta" ) @@ -73,16 +74,28 @@ type ComponentHandler interface { SetCreateOnly() } +// ComponentHandlerOption configures a componentHandler. +type ComponentHandlerOption func(*componentHandler) + +// WithContext supplies the operator.Context passed to registered render patches. +func WithContext(ctx operator.Context) ComponentHandlerOption { + return func(c *componentHandler) { c.modCtx = ctx } +} + // cr is allowed to be nil in the case we don't want to put ownership on a resource, // this is useful for CRD management so that they are not removed automatically. -func NewComponentHandler(log logr.Logger, cli client.Client, scheme *runtime.Scheme, cr metav1.Object) ComponentHandler { - return &componentHandler{ +func NewComponentHandler(log logr.Logger, cli client.Client, scheme *runtime.Scheme, cr metav1.Object, opts ...ComponentHandlerOption) ComponentHandler { + h := &componentHandler{ client: cli, scheme: scheme, cr: cr, log: log, apiGroupEnvs: apigroup.EnvVars(), } + for _, o := range opts { + o(h) + } + return h } type componentHandler struct { @@ -92,6 +105,7 @@ type componentHandler struct { log logr.Logger createOnly bool apiGroupEnvs []v1.EnvVar + modCtx operator.Context } func (c *componentHandler) SetCreateOnly() { @@ -459,6 +473,9 @@ func (c *componentHandler) CreateOrUpdateOrDelete(ctx context.Context, component var cronJobs []types.NamespacedName objsToCreate, objsToDelete := component.Objects() + if named, ok := component.(render.Named); ok { + objsToCreate = operator.ApplyPatches(named.Name(), c.modCtx, objsToCreate) + } osType := component.SupportedOSType() if len(c.apiGroupEnvs) > 0 { diff --git a/pkg/controller/utils/component_test.go b/pkg/controller/utils/component_test.go index d2e18c8482..d4b34cb871 100644 --- a/pkg/controller/utils/component_test.go +++ b/pkg/controller/utils/component_test.go @@ -46,6 +46,7 @@ import ( "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/controller/status" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/operator" "github.com/tigera/operator/pkg/render" rmeta "github.com/tigera/operator/pkg/render/common/meta" ) @@ -2457,3 +2458,45 @@ func (mc *mockClient) RESTMapper() restMeta.RESTMapper { func (mc *mockClient) SubResource(subResource string) client.SubResourceClient { panic("SubResource not implemented in mockClient") } + +var _ = Describe("componentHandler patch application", func() { + AfterEach(func() { + operator.ResetForTest() + }) + + It("applies registered patches to a named component before create", func() { + operator.Patch("fake", func(ctx operator.Context, objs []client.Object) []client.Object { + cm := objs[0].(*corev1.ConfigMap) + cm.Data = map[string]string{"patched": "yes"} + return objs + }) + + s := runtime.NewScheme() + Expect(apis.AddToScheme(s, false)).NotTo(HaveOccurred()) + Expect(corev1.SchemeBuilder.AddToScheme(s)).NotTo(HaveOccurred()) + + c := ctrlrfake.DefaultFakeClientBuilder(s).Build() + handler := NewComponentHandler(logf.Log, c, s, nil) + comp := &namedFakeComponent{name: "fake", obj: &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "cm", Namespace: "default"}, + }} + + Expect(handler.CreateOrUpdateOrDelete(context.Background(), comp, nil)).NotTo(HaveOccurred()) + + got := &corev1.ConfigMap{} + Expect(c.Get(context.Background(), client.ObjectKey{Name: "cm", Namespace: "default"}, got)).NotTo(HaveOccurred()) + Expect(got.Data).To(HaveKeyWithValue("patched", "yes")) + }) +}) + +type namedFakeComponent struct { + name string + obj client.Object +} + +func (f *namedFakeComponent) Name() string { return f.name } +func (f *namedFakeComponent) ResolveImages(*operatorv1.ImageSet) error { return nil } +func (f *namedFakeComponent) Objects() ([]client.Object, []client.Object) { return []client.Object{f.obj}, nil } +func (f *namedFakeComponent) Ready() bool { return true } +func (f *namedFakeComponent) SupportedOSType() rmeta.OSType { return rmeta.OSTypeLinux } From cc62d3cb3c39b098799f234d2fd88b52aaa3962f Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Thu, 28 May 2026 16:49:29 -0700 Subject: [PATCH 06/38] operator: add installation controller extension registry --- pkg/operator/extension.go | 59 ++++++++++++++++++++++++++++++++++ pkg/operator/extension_test.go | 42 ++++++++++++++++++++++++ 2 files changed, 101 insertions(+) create mode 100644 pkg/operator/extension.go create mode 100644 pkg/operator/extension_test.go diff --git a/pkg/operator/extension.go b/pkg/operator/extension.go new file mode 100644 index 0000000000..de5080c45d --- /dev/null +++ b/pkg/operator/extension.go @@ -0,0 +1,59 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package operator + +import ( + "context" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/tls/certificatemanagement" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// InstallationPrep is the input to an InstallationExtension's Prepare. It holds +// the generically-gathered reconcile state the extension needs to do its +// side-effecting work (create certs, assemble the trusted bundle). +type InstallationPrep struct { + Ctx context.Context + Client client.Client + Installation *operatorv1.InstallationSpec + FelixConfiguration *v3.FelixConfiguration + CertificateManager certificatemanager.CertificateManager + TrustedBundle certificatemanagement.TrustedBundle + ClusterDomain string +} + +// InstallationExtension is the enterprise hook for the installation controller. +// Prepare runs controller-side before rendering. It performs work modifiers +// can't (cluster side effects, fetching/creating certificates) and may abort +// the reconcile by returning an error. It returns the Context handed to the +// render patches; on the Calico variant it should return an empty Context and +// nil error. +type InstallationExtension interface { + Prepare(p InstallationPrep) (Context, error) +} + +var installationExtension InstallationExtension + +// RegisterInstallationExtension registers the installation controller extension. +func RegisterInstallationExtension(e InstallationExtension) { installationExtension = e } + +// GetInstallationExtension returns the registered extension, or nil. +func GetInstallationExtension() InstallationExtension { return installationExtension } + +// ResetExtensionsForTest clears registered extensions. Test-only. +func ResetExtensionsForTest() { installationExtension = nil } diff --git a/pkg/operator/extension_test.go b/pkg/operator/extension_test.go new file mode 100644 index 0000000000..e3ecd95592 --- /dev/null +++ b/pkg/operator/extension_test.go @@ -0,0 +1,42 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package operator_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/tigera/operator/pkg/operator" +) + +var _ = Describe("controller extensions", func() { + AfterEach(func() { operator.ResetExtensionsForTest() }) + + It("returns the registered installation extension", func() { + ext := &fakeInstallationExtension{} + operator.RegisterInstallationExtension(ext) + Expect(operator.GetInstallationExtension()).To(BeIdenticalTo(ext)) + }) + + It("returns nil when none is registered", func() { + Expect(operator.GetInstallationExtension()).To(BeNil()) + }) +}) + +type fakeInstallationExtension struct{} + +func (f *fakeInstallationExtension) Prepare(_ operator.InstallationPrep) (operator.Context, error) { + return operator.Context{}, nil +} From e8c588f4aaf3dc4d2df914d13b4bb2a16c1a72ca Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Thu, 28 May 2026 16:50:25 -0700 Subject: [PATCH 07/38] render: typha implements Named --- pkg/render/typha.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/render/typha.go b/pkg/render/typha.go index cc352771dc..49c3f13917 100644 --- a/pkg/render/typha.go +++ b/pkg/render/typha.go @@ -113,6 +113,8 @@ func (c *typhaComponent) SupportedOSType() rmeta.OSType { return rmeta.OSTypeLinux } +func (c *typhaComponent) Name() string { return ComponentNameTypha } + func (c *typhaComponent) Objects() ([]client.Object, []client.Object) { objs := []client.Object{ c.typhaServiceAccount(), From f7b28f35adefeaf72c1b385565ba087996f33bda Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Thu, 28 May 2026 23:17:12 -0700 Subject: [PATCH 08/38] enterprise: move typha variant branches into a modifier Pulls the enterprise RBAC extra-rules and MULTI_INTERFACE_MODE env branches out of pkg/render/typha.go into a new pkg/enterprise package. The enterprise package registers a patch via operator.Patch on startup; pkg/render/typha.go now has zero IsEnterprise branches. --- pkg/enterprise/enterprise_suite_test.go | 27 ++++++++ pkg/enterprise/register.go | 22 +++++++ pkg/enterprise/typha.go | 66 +++++++++++++++++++ pkg/enterprise/typha_test.go | 88 +++++++++++++++++++++++++ pkg/render/typha.go | 29 -------- 5 files changed, 203 insertions(+), 29 deletions(-) create mode 100644 pkg/enterprise/enterprise_suite_test.go create mode 100644 pkg/enterprise/register.go create mode 100644 pkg/enterprise/typha.go create mode 100644 pkg/enterprise/typha_test.go diff --git a/pkg/enterprise/enterprise_suite_test.go b/pkg/enterprise/enterprise_suite_test.go new file mode 100644 index 0000000000..e7cfab281b --- /dev/null +++ b/pkg/enterprise/enterprise_suite_test.go @@ -0,0 +1,27 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enterprise_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestEnterprise(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "pkg/enterprise Suite") +} diff --git a/pkg/enterprise/register.go b/pkg/enterprise/register.go new file mode 100644 index 0000000000..88315d93ba --- /dev/null +++ b/pkg/enterprise/register.go @@ -0,0 +1,22 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enterprise + +// Register wires all in-repo enterprise modifiers and controller extensions +// into the operator registries. Called once at process startup. After the +// monorepo split this is what calico-private's main will do instead. +func Register() { + registerTypha() +} diff --git a/pkg/enterprise/typha.go b/pkg/enterprise/typha.go new file mode 100644 index 0000000000..8deef34b28 --- /dev/null +++ b/pkg/enterprise/typha.go @@ -0,0 +1,66 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enterprise + +import ( + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/tigera/operator/pkg/operator" + "github.com/tigera/operator/pkg/render" +) + +func registerTypha() { + operator.Patch(render.ComponentNameTypha, patchTypha) +} + +func patchTypha(ctx operator.Context, objs []client.Object) []client.Object { + if ctx.Installation == nil || !ctx.Installation.Variant.IsEnterprise() { + return objs + } + + if role, ok := operator.FindObject[*rbacv1.ClusterRole](objs, "calico-typha"); ok { + role.Rules = append(role.Rules, rbacv1.PolicyRule{ + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{ + "bfdconfigurations", + "deeppacketinspections", + "egressgatewaypolicies", + "externalnetworks", + "licensekeys", + "networks", + "packetcaptures", + "remoteclusterconfigurations", + }, + Verbs: []string{"get", "list", "watch"}, + }) + } + + if dep, ok := operator.FindObject[*appsv1.Deployment](objs, "calico-typha"); ok { + net := ctx.Installation.CalicoNetwork + if net != nil && net.MultiInterfaceMode != nil { + for i := range dep.Spec.Template.Spec.Containers { + if dep.Spec.Template.Spec.Containers[i].Name == render.TyphaContainerName { + c := &dep.Spec.Template.Spec.Containers[i] + c.Env = append(c.Env, corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: net.MultiInterfaceMode.Value()}) + } + } + } + } + + return objs +} diff --git a/pkg/enterprise/typha_test.go b/pkg/enterprise/typha_test.go new file mode 100644 index 0000000000..67330b7539 --- /dev/null +++ b/pkg/enterprise/typha_test.go @@ -0,0 +1,88 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enterprise_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/enterprise" + "github.com/tigera/operator/pkg/operator" + "github.com/tigera/operator/pkg/render" +) + +var _ = Describe("typha enterprise modifier", func() { + BeforeEach(func() { enterprise.Register() }) + AfterEach(func() { + operator.ResetForTest() + operator.ResetExtensionsForTest() + }) + + multiMode := operatorv1.MultiInterfaceModeMultus + + newObjs := func() []client.Object { + return []client.Object{ + &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-typha"}}, + &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "calico-typha"}, + Spec: appsv1.DeploymentSpec{Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: render.TyphaContainerName}}, + }}}, + }, + } + } + + It("adds enterprise RBAC and MULTI_INTERFACE_MODE for the enterprise variant", func() { + ctx := operator.Context{Installation: &operatorv1.InstallationSpec{ + Variant: operatorv1.TigeraSecureEnterprise, + CalicoNetwork: &operatorv1.CalicoNetworkSpec{MultiInterfaceMode: &multiMode}, + }} + out := operator.ApplyPatches(render.ComponentNameTypha, ctx, newObjs()) + + role := out[0].(*rbacv1.ClusterRole) + Expect(role.Rules).To(ContainElement(HaveField("Resources", ContainElement("licensekeys")))) + + dep := out[1].(*appsv1.Deployment) + var c *corev1.Container + for i := range dep.Spec.Template.Spec.Containers { + if dep.Spec.Template.Spec.Containers[i].Name == render.TyphaContainerName { + c = &dep.Spec.Template.Spec.Containers[i] + } + } + Expect(c.Env).To(ContainElement(corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: multiMode.Value()})) + }) + + It("is a no-op for the Calico variant", func() { + ctx := operator.Context{Installation: &operatorv1.InstallationSpec{ + Variant: operatorv1.Calico, + CalicoNetwork: &operatorv1.CalicoNetworkSpec{MultiInterfaceMode: &multiMode}, + }} + out := operator.ApplyPatches(render.ComponentNameTypha, ctx, newObjs()) + Expect(out[0].(*rbacv1.ClusterRole).Rules).To(BeEmpty()) + dep := out[1].(*appsv1.Deployment) + Expect(dep.Spec.Template.Spec.Containers[0].Env).To(BeEmpty()) + }) + + It("does not panic on a zero Context (nil Installation)", func() { + out := operator.ApplyPatches(render.ComponentNameTypha, operator.Context{}, newObjs()) + Expect(out[0].(*rbacv1.ClusterRole).Rules).To(BeEmpty()) + }) +}) diff --git a/pkg/render/typha.go b/pkg/render/typha.go index 49c3f13917..84cc8dab9f 100644 --- a/pkg/render/typha.go +++ b/pkg/render/typha.go @@ -353,26 +353,6 @@ func (c *typhaComponent) typhaRole() *rbacv1.ClusterRole { }, }, } - if c.cfg.Installation.Variant.IsEnterprise() { - extraRules := []rbacv1.PolicyRule{ - { - // Tigera Secure needs to be able to read licenses, and config. - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{ - "bfdconfigurations", - "deeppacketinspections", - "egressgatewaypolicies", - "externalnetworks", - "licensekeys", - "networks", - "packetcaptures", - "remoteclusterconfigurations", - }, - Verbs: []string{"get", "list", "watch"}, - }, - } - role.Rules = append(role.Rules, extraRules...) - } if c.cfg.Installation.KubernetesProvider.IsOpenShift() { role.Rules = append(role.Rules, rbacv1.PolicyRule{ APIGroups: []string{"security.openshift.io"}, @@ -630,15 +610,6 @@ func (c *typhaComponent) typhaEnvVars(typhaSecret certificatemanagement.KeyPairI typhaEnv = append(typhaEnv, corev1.EnvVar{Name: "FELIX_INTERFACEPREFIX", Value: "azv"}) } - if c.cfg.Installation.Variant.IsEnterprise() { - if c.cfg.Installation.CalicoNetwork != nil && c.cfg.Installation.CalicoNetwork.MultiInterfaceMode != nil { - typhaEnv = append(typhaEnv, corev1.EnvVar{ - Name: "MULTI_INTERFACE_MODE", - Value: c.cfg.Installation.CalicoNetwork.MultiInterfaceMode.Value(), - }) - } - } - // If host-local IPAM is in use, we need to configure typha to use the Kubernetes pod CIDR. cni := c.cfg.Installation.CNI if cni != nil && cni.IPAM != nil && cni.IPAM.Type == operatorv1.IPAMPluginHostLocal { From e78d6336b3eb93d984c683522a967c4bd10d1c80 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Fri, 29 May 2026 08:09:13 -0700 Subject: [PATCH 09/38] installation: register enterprise modifiers and pass render Context Calls enterprise.Register() at startup so the typha modifier is wired in. Builds an operator.Context in the installation reconciler and passes it to the component handler so registered modifiers receive reconcile-derived state. --- cmd/main.go | 6 ++++++ pkg/controller/installation/core_controller.go | 12 +++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/cmd/main.go b/cmd/main.go index 7b1117b355..b0aee8fb59 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -41,6 +41,7 @@ import ( "github.com/tigera/operator/pkg/controller/options" "github.com/tigera/operator/pkg/controller/utils" "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/enterprise" "github.com/tigera/operator/pkg/imports/admission" "github.com/tigera/operator/pkg/imports/crds" "github.com/tigera/operator/pkg/render" @@ -517,6 +518,11 @@ If a value other than 'all' is specified, the first CRD with a prefix of the spe os.Exit(1) } + // Wire in-repo enterprise modifiers and extensions into the operator + // registries. After the monorepo split this call moves to calico-private's + // main. + enterprise.Register() + err = controller.AddToManager(mgr, options) if err != nil { setupLog.Error(err, "unable to create controllers") diff --git a/pkg/controller/installation/core_controller.go b/pkg/controller/installation/core_controller.go index 89acae7416..a966a02a0e 100644 --- a/pkg/controller/installation/core_controller.go +++ b/pkg/controller/installation/core_controller.go @@ -63,6 +63,7 @@ import ( "github.com/tigera/operator/pkg/active" "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/operator" "github.com/tigera/operator/pkg/controller/certificatemanager" "github.com/tigera/operator/pkg/controller/ippool" "github.com/tigera/operator/pkg/controller/k8sapi" @@ -1294,8 +1295,17 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile nodeAppArmorProfile = val } + // Build the render Context carrying reconcile-derived state for any + // registered modifiers (e.g. enterprise typha branches). + modCtx := operator.Context{ + Installation: &instance.Spec, + FelixConfiguration: felixConfiguration, + ClusterDomain: r.clusterDomain, + TrustedBundle: typhaNodeTLS.TrustedBundle, + } + // Create a component handler to create or update the rendered components. - handler := r.newComponentHandler(log, r.client, r.scheme, instance) + handler := r.newComponentHandler(log, r.client, r.scheme, instance, utils.WithContext(modCtx)) // Render namespaces first - this ensures that any other controllers blocked on namespace existence can proceed. namespaceCfg := &render.NamespaceConfiguration{ From 2e48ab3ab8eb84ed57473c9f80a395e367b35ff0 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Fri, 29 May 2026 09:03:04 -0700 Subject: [PATCH 10/38] enterprise: route node image selection through ResolveImage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extracts the image override registry into a leaf pkg/imageoverride package (no render/operator transitive deps) to avoid the render→operator import cycle. operator.OverrideImage/ResolveImage now delegate there. Registers the enterprise node image override in pkg/enterprise. Removes the IsEnterprise image switch from render/node.go; FIPS handling is preserved via a post-resolve check. --- pkg/enterprise/node.go | 31 ++++++++++++++++++ pkg/enterprise/node_test.go | 43 ++++++++++++++++++++++++ pkg/enterprise/register.go | 1 + pkg/imageoverride/imageoverride.go | 51 +++++++++++++++++++++++++++++ pkg/operator/image.go | 14 +++----- pkg/operator/patch.go | 8 +++-- pkg/render/enterprise_setup_test.go | 29 ++++++++++++++++ pkg/render/node.go | 20 +++++++---- 8 files changed, 178 insertions(+), 19 deletions(-) create mode 100644 pkg/enterprise/node.go create mode 100644 pkg/enterprise/node_test.go create mode 100644 pkg/imageoverride/imageoverride.go create mode 100644 pkg/render/enterprise_setup_test.go diff --git a/pkg/enterprise/node.go b/pkg/enterprise/node.go new file mode 100644 index 0000000000..039303681f --- /dev/null +++ b/pkg/enterprise/node.go @@ -0,0 +1,31 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enterprise + +import ( + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/operator" + "github.com/tigera/operator/pkg/render" +) + +func registerNode() { + operator.OverrideImage(render.ComponentNameNode, func(in *operatorv1.InstallationSpec) (components.Component, bool) { + if !in.Variant.IsEnterprise() { + return components.Component{}, false + } + return components.ComponentTigeraNode, true + }) +} diff --git a/pkg/enterprise/node_test.go b/pkg/enterprise/node_test.go new file mode 100644 index 0000000000..d12d9812f2 --- /dev/null +++ b/pkg/enterprise/node_test.go @@ -0,0 +1,43 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enterprise_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/enterprise" + "github.com/tigera/operator/pkg/operator" +) + +var _ = Describe("node enterprise image override", func() { + BeforeEach(func() { enterprise.Register() }) + AfterEach(func() { + operator.ResetForTest() + operator.ResetExtensionsForTest() + }) + + It("selects the enterprise node image for the enterprise variant", func() { + ent := &operatorv1.InstallationSpec{Variant: operatorv1.TigeraSecureEnterprise} + Expect(operator.ResolveImage("node", components.ComponentCalicoNode, ent)).To(Equal(components.ComponentTigeraNode)) + }) + + It("leaves the default in place for the Calico variant", func() { + oss := &operatorv1.InstallationSpec{Variant: operatorv1.Calico} + Expect(operator.ResolveImage("node", components.ComponentCalicoNode, oss)).To(Equal(components.ComponentCalicoNode)) + }) +}) diff --git a/pkg/enterprise/register.go b/pkg/enterprise/register.go index 88315d93ba..d32e8c66b1 100644 --- a/pkg/enterprise/register.go +++ b/pkg/enterprise/register.go @@ -19,4 +19,5 @@ package enterprise // monorepo split this is what calico-private's main will do instead. func Register() { registerTypha() + registerNode() } diff --git a/pkg/imageoverride/imageoverride.go b/pkg/imageoverride/imageoverride.go new file mode 100644 index 0000000000..bf8950cf84 --- /dev/null +++ b/pkg/imageoverride/imageoverride.go @@ -0,0 +1,51 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package imageoverride is a leaf package (no render/operator dependencies) +// that holds the image override registry. Both pkg/operator and pkg/render +// import it to avoid the render→operator→render import cycle. +package imageoverride + +import ( + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/components" +) + +// Override selects the component image to use for an installation, returning +// false to decline (leaving the default in place). +type Override func(in *operatorv1.InstallationSpec) (components.Component, bool) + +var registry = map[string]Override{} + +// Register stores fn under key. The key is the render component's image +// identifier (e.g. "node"). +func Register(key string, fn Override) { + registry[key] = fn +} + +// Resolve returns the override registered for key if it applies to in, +// otherwise def. +func Resolve(key string, def components.Component, in *operatorv1.InstallationSpec) components.Component { + if fn, ok := registry[key]; ok { + if c, override := fn(in); override { + return c + } + } + return def +} + +// ResetForTest clears the registry. Test-only. +func ResetForTest() { + registry = map[string]Override{} +} diff --git a/pkg/operator/image.go b/pkg/operator/image.go index b253611254..c6de3b917a 100644 --- a/pkg/operator/image.go +++ b/pkg/operator/image.go @@ -17,28 +17,22 @@ package operator import ( operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/imageoverride" ) // ImageOverride returns the component image to use for an installation, and // false to decline (leaving the default in place). Implementations self-gate // on in.Variant. -type ImageOverride func(in *operatorv1.InstallationSpec) (components.Component, bool) - -var imageOverrides = map[string]ImageOverride{} +type ImageOverride = imageoverride.Override // OverrideImage registers an image override under key. The key is the render // component's image identifier (e.g. "node"). func OverrideImage(key string, fn ImageOverride) { - imageOverrides[key] = fn + imageoverride.Register(key, fn) } // ResolveImage returns the override registered for key if it applies to in, // otherwise def. Render components call this inside ResolveImages. func ResolveImage(key string, def components.Component, in *operatorv1.InstallationSpec) components.Component { - if fn, ok := imageOverrides[key]; ok { - if c, override := fn(in); override { - return c - } - } - return def + return imageoverride.Resolve(key, def, in) } diff --git a/pkg/operator/patch.go b/pkg/operator/patch.go index 94721a76bd..bce2ff783c 100644 --- a/pkg/operator/patch.go +++ b/pkg/operator/patch.go @@ -14,7 +14,11 @@ package operator -import "sigs.k8s.io/controller-runtime/pkg/client" +import ( + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/tigera/operator/pkg/imageoverride" +) // PatchFunc post-processes the objects a render component produced. It may // mutate matched objects and/or append additional objects, and must return the @@ -51,5 +55,5 @@ func FindObject[T client.Object](objs []client.Object, name string) (T, bool) { // ResetForTest clears all registries. Test-only. func ResetForTest() { patches = map[string][]PatchFunc{} - imageOverrides = map[string]ImageOverride{} + imageoverride.ResetForTest() } diff --git a/pkg/render/enterprise_setup_test.go b/pkg/render/enterprise_setup_test.go new file mode 100644 index 0000000000..5f464c950b --- /dev/null +++ b/pkg/render/enterprise_setup_test.go @@ -0,0 +1,29 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package render_test + +import ( + . "github.com/onsi/ginkgo/v2" + + "github.com/tigera/operator/pkg/enterprise" +) + +// The render suite asserts enterprise-variant output for components whose +// variant-specific behavior now lives in registered modifiers/overrides +// (e.g. the node image). Register them once so the suite exercises the same +// integrated behavior the operator binary produces. +var _ = BeforeSuite(func() { + enterprise.Register() +}) diff --git a/pkg/render/node.go b/pkg/render/node.go index 638c95b351..07e1be94db 100644 --- a/pkg/render/node.go +++ b/pkg/render/node.go @@ -34,6 +34,7 @@ import ( "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/controller/k8sapi" + "github.com/tigera/operator/pkg/imageoverride" "github.com/tigera/operator/pkg/controller/migration" rcomp "github.com/tigera/operator/pkg/render/common/components" "github.com/tigera/operator/pkg/render/common/configmap" @@ -184,14 +185,15 @@ func (c *nodeComponent) ResolveImages(is *operatorv1.ImageSet) error { } c.calicoImage = appendIfErr(components.GetReference(components.CombinedCalicoImage(c.cfg.Installation), reg, path, prefix, is)) - switch { - case c.cfg.Installation.Variant.IsEnterprise(): - c.nodeImage = appendIfErr(components.GetReference(components.ComponentTigeraNode, reg, path, prefix, is)) - case operatorv1.IsFIPSModeEnabled(c.cfg.Installation.FIPSMode): - c.nodeImage = appendIfErr(components.GetReference(components.ComponentCalicoNodeFIPS, reg, path, prefix, is)) - default: - c.nodeImage = appendIfErr(components.GetReference(components.ComponentCalicoNode, reg, path, prefix, is)) + nodeImage := imageoverride.Resolve(ComponentNameNode, components.ComponentCalicoNode, c.cfg.Installation) + if operatorv1.IsFIPSModeEnabled(c.cfg.Installation.FIPSMode) { + // FIPS only applies to the Calico variant; the enterprise override (if + // registered) has already replaced nodeImage for the enterprise variant. + if nodeImage == components.ComponentCalicoNode { + nodeImage = components.ComponentCalicoNodeFIPS + } } + c.nodeImage = appendIfErr(components.GetReference(nodeImage, reg, path, prefix, is)) if len(errMsgs) != 0 { return fmt.Errorf("%s", strings.Join(errMsgs, ",")) @@ -203,6 +205,10 @@ func (c *nodeComponent) SupportedOSType() rmeta.OSType { return rmeta.OSTypeLinux } +func (c *nodeComponent) Name() string { + return ComponentNameNode +} + func (c *nodeComponent) Objects() ([]client.Object, []client.Object) { objs := []client.Object{ c.nodeServiceAccount(), From 0506f4c1c127d3e31b2d1041aed985520a1db0fb Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Fri, 29 May 2026 09:05:12 -0700 Subject: [PATCH 11/38] render: fix node.go import ordering --- pkg/render/node.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/render/node.go b/pkg/render/node.go index 07e1be94db..9c335d9ea4 100644 --- a/pkg/render/node.go +++ b/pkg/render/node.go @@ -34,8 +34,8 @@ import ( "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/controller/k8sapi" - "github.com/tigera/operator/pkg/imageoverride" "github.com/tigera/operator/pkg/controller/migration" + "github.com/tigera/operator/pkg/imageoverride" rcomp "github.com/tigera/operator/pkg/render/common/components" "github.com/tigera/operator/pkg/render/common/configmap" rmeta "github.com/tigera/operator/pkg/render/common/meta" From f240c2d1518c29c33a400111b9bc875dbc42f230 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Fri, 29 May 2026 09:28:24 -0700 Subject: [PATCH 12/38] enterprise: move node-prometheus cert setup into an InstallationExtension The OSS installation controller no longer directly creates the node-prometheus keypair or fetches the prometheus/esgw certs. Those are now handled by a registered InstallationExtension in pkg/enterprise. Port value derivation and the kube-controller TLS block remain in the OSS controller unchanged. --- .../installation/core_controller.go | 79 ++++++----------- .../installation_controller_suite_test.go | 6 ++ pkg/enterprise/installation.go | 80 +++++++++++++++++ pkg/enterprise/installation_test.go | 85 +++++++++++++++++++ pkg/enterprise/register.go | 1 + 5 files changed, 199 insertions(+), 52 deletions(-) create mode 100644 pkg/enterprise/installation.go create mode 100644 pkg/enterprise/installation_test.go diff --git a/pkg/controller/installation/core_controller.go b/pkg/controller/installation/core_controller.go index a966a02a0e..bf92bf3e2f 100644 --- a/pkg/controller/installation/core_controller.go +++ b/pkg/controller/installation/core_controller.go @@ -80,8 +80,7 @@ import ( "github.com/tigera/operator/pkg/imports/crds" "github.com/tigera/operator/pkg/render" rcertificatemanagement "github.com/tigera/operator/pkg/render/certificatemanagement" - relasticsearch "github.com/tigera/operator/pkg/render/common/elasticsearch" - "github.com/tigera/operator/pkg/render/common/networkpolicy" +"github.com/tigera/operator/pkg/render/common/networkpolicy" "github.com/tigera/operator/pkg/render/common/resourcequota" "github.com/tigera/operator/pkg/render/goldmane" "github.com/tigera/operator/pkg/render/kubecontrollers" @@ -1202,62 +1201,47 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile // nodeReporterMetricsPort is a port used in Enterprise to host internal metrics. // Operator is responsible for creating a service which maps to that port. - // Here, we'll check the default felixconfiguration to see if the user is specifying - // a non-default port, and use that value if they are. nodeReporterMetricsPort := defaultNodeReporterPort - var nodePrometheusTLS certificatemanagement.KeyPairInterface - calicoVersion := components.CalicoRelease - felixPrometheusMetricsPort := defaultFelixMetricsDefaultPort - + calicoVersion := components.CalicoRelease if instance.Spec.Variant.IsEnterprise() { - - // Determine the port to use for nodeReporter metrics. + calicoVersion = components.EnterpriseRelease if felixConfiguration.Spec.PrometheusReporterPort != nil { nodeReporterMetricsPort = *felixConfiguration.Spec.PrometheusReporterPort } - if nodeReporterMetricsPort == 0 { - err := errors.New("felixConfiguration prometheusReporterPort=0 not supported") - r.status.SetDegraded(operatorv1.InvalidConfigurationError, "invalid metrics port", err, reqLogger) - return reconcile.Result{}, err - } - if felixConfiguration.Spec.PrometheusMetricsPort != nil { felixPrometheusMetricsPort = *felixConfiguration.Spec.PrometheusMetricsPort } + } - nodePrometheusTLS, err = certificateManager.GetOrCreateKeyPair(r.client, render.NodePrometheusTLSServerSecret, common.OperatorNamespace(), dns.GetServiceDNSNames(render.CalicoNodeMetricsService, common.CalicoNamespace, r.clusterDomain)) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceCreateError, "Error creating TLS certificate", err, reqLogger) - return reconcile.Result{}, err - } - if nodePrometheusTLS != nil { - typhaNodeTLS.TrustedBundle.AddCertificates(nodePrometheusTLS) - } - prometheusClientCert, err := certificateManager.GetCertificate(r.client, monitor.PrometheusClientTLSSecretName, common.OperatorNamespace()) - if err != nil { - r.status.SetDegraded(operatorv1.CertificateError, "Unable to fetch prometheus certificate", err, reqLogger) - return reconcile.Result{}, err - } - if prometheusClientCert != nil { - typhaNodeTLS.TrustedBundle.AddCertificates(prometheusClientCert) - } - - // es-kube-controllers needs to trust the ESGW certificate. We'll fetch it here and add it to the trusted bundle. - // Note that although we're adding this to the typhaNodeTLS trusted bundle, it will be used by es-kube-controllers. This is because - // all components within this namespace share a trusted CA bundle. This is necessary because prior to v3.13 secrets were not signed by - // a single CA so we need to include each individually. - esgwCertificate, err := certificateManager.GetCertificate(r.client, relasticsearch.PublicCertSecret, common.OperatorNamespace()) + // Run the registered installation extension. For the enterprise variant this + // validates config and creates the node-prometheus certificate, adding it (and + // the prometheus/esgw certs) to the trusted bundle. Returns the render Context + // consumed by registered modifiers. + var modCtx operator.Context + if ext := operator.GetInstallationExtension(); ext != nil { + modCtx, err = ext.Prepare(operator.InstallationPrep{ + Ctx: ctx, + Client: r.client, + Installation: &instance.Spec, + FelixConfiguration: felixConfiguration, + CertificateManager: certificateManager, + TrustedBundle: typhaNodeTLS.TrustedBundle, + ClusterDomain: r.clusterDomain, + }) if err != nil { - r.status.SetDegraded(operatorv1.CertificateError, fmt.Sprintf("Failed to retrieve / validate %s", relasticsearch.PublicCertSecret), err, reqLogger) + r.status.SetDegraded(operatorv1.ResourceCreateError, "Error preparing enterprise installation", err, reqLogger) return reconcile.Result{}, err } - if esgwCertificate != nil { - typhaNodeTLS.TrustedBundle.AddCertificates(esgwCertificate) + } else { + modCtx = operator.Context{ + Installation: &instance.Spec, + FelixConfiguration: felixConfiguration, + ClusterDomain: r.clusterDomain, + TrustedBundle: typhaNodeTLS.TrustedBundle, } - - calicoVersion = components.EnterpriseRelease } + nodePrometheusTLS := modCtx.NodePrometheusTLS kubeControllersMetricsPort, err := utils.GetKubeControllerMetricsPort(ctx, r.client) if err != nil { @@ -1295,15 +1279,6 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile nodeAppArmorProfile = val } - // Build the render Context carrying reconcile-derived state for any - // registered modifiers (e.g. enterprise typha branches). - modCtx := operator.Context{ - Installation: &instance.Spec, - FelixConfiguration: felixConfiguration, - ClusterDomain: r.clusterDomain, - TrustedBundle: typhaNodeTLS.TrustedBundle, - } - // Create a component handler to create or update the rendered components. handler := r.newComponentHandler(log, r.client, r.scheme, instance, utils.WithContext(modCtx)) diff --git a/pkg/controller/installation/installation_controller_suite_test.go b/pkg/controller/installation/installation_controller_suite_test.go index 4924f3468b..778ee91f8f 100644 --- a/pkg/controller/installation/installation_controller_suite_test.go +++ b/pkg/controller/installation/installation_controller_suite_test.go @@ -25,8 +25,14 @@ import ( clientfeaturestesting "k8s.io/client-go/features/testing" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" + + "github.com/tigera/operator/pkg/enterprise" ) +var _ = ginkgo.BeforeSuite(func() { + enterprise.Register() +}) + func TestInstallation(t *testing.T) { // Disable WatchListClient for tests. In client-go v0.35+, this feature defaults to true and // causes informers to wait for bookmark events that fake clients never send, leading to timeouts. diff --git a/pkg/enterprise/installation.go b/pkg/enterprise/installation.go new file mode 100644 index 0000000000..7b0abaf4b5 --- /dev/null +++ b/pkg/enterprise/installation.go @@ -0,0 +1,80 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enterprise + +import ( + "errors" + "fmt" + + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/render/monitor" + "github.com/tigera/operator/pkg/operator" + "github.com/tigera/operator/pkg/render" + relasticsearch "github.com/tigera/operator/pkg/render/common/elasticsearch" +) + +type installationExtension struct{} + +func registerInstallation() { + operator.RegisterInstallationExtension(&installationExtension{}) +} + +func (e *installationExtension) Prepare(p operator.InstallationPrep) (operator.Context, error) { + ctx := operator.Context{ + Installation: p.Installation, + FelixConfiguration: p.FelixConfiguration, + ClusterDomain: p.ClusterDomain, + TrustedBundle: p.TrustedBundle, + } + if !p.Installation.Variant.IsEnterprise() { + return ctx, nil + } + + // Reject the unsupported zero reporter port. (Port value derivation stays in + // the OSS controller; only validation moves here.) + if p.FelixConfiguration.Spec.PrometheusReporterPort != nil && *p.FelixConfiguration.Spec.PrometheusReporterPort == 0 { + return ctx, errors.New("felixConfiguration prometheusReporterPort=0 not supported") + } + + nodePrometheusTLS, err := p.CertificateManager.GetOrCreateKeyPair( + p.Client, render.NodePrometheusTLSServerSecret, common.OperatorNamespace(), + dns.GetServiceDNSNames(render.CalicoNodeMetricsService, common.CalicoNamespace, p.ClusterDomain)) + if err != nil { + return ctx, fmt.Errorf("error creating node prometheus TLS certificate: %w", err) + } + if nodePrometheusTLS != nil { + p.TrustedBundle.AddCertificates(nodePrometheusTLS) + } + ctx.NodePrometheusTLS = nodePrometheusTLS + + prometheusClientCert, err := p.CertificateManager.GetCertificate(p.Client, monitor.PrometheusClientTLSSecretName, common.OperatorNamespace()) + if err != nil { + return ctx, fmt.Errorf("unable to fetch prometheus certificate: %w", err) + } + if prometheusClientCert != nil { + p.TrustedBundle.AddCertificates(prometheusClientCert) + } + + esgwCertificate, err := p.CertificateManager.GetCertificate(p.Client, relasticsearch.PublicCertSecret, common.OperatorNamespace()) + if err != nil { + return ctx, fmt.Errorf("failed to retrieve / validate %s: %w", relasticsearch.PublicCertSecret, err) + } + if esgwCertificate != nil { + p.TrustedBundle.AddCertificates(esgwCertificate) + } + + return ctx, nil +} diff --git a/pkg/enterprise/installation_test.go b/pkg/enterprise/installation_test.go new file mode 100644 index 0000000000..dffe6a7470 --- /dev/null +++ b/pkg/enterprise/installation_test.go @@ -0,0 +1,85 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enterprise_test + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/apis" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/controller/certificatemanager" + ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/enterprise" + "github.com/tigera/operator/pkg/operator" + "k8s.io/apimachinery/pkg/runtime" +) + +var _ = Describe("installation enterprise extension", func() { + BeforeEach(func() { enterprise.Register() }) + AfterEach(func() { + operator.ResetForTest() + operator.ResetExtensionsForTest() + }) + + It("rejects a zero prometheus reporter port", func() { + port := 0 + p := newPrep(operatorv1.TigeraSecureEnterprise) + p.FelixConfiguration = &v3.FelixConfiguration{Spec: v3.FelixConfigurationSpec{PrometheusReporterPort: &port}} + _, err := operator.GetInstallationExtension().Prepare(p) + Expect(err).To(HaveOccurred()) + }) + + It("creates the node prometheus keypair for the enterprise variant", func() { + p := newPrep(operatorv1.TigeraSecureEnterprise) + p.FelixConfiguration = &v3.FelixConfiguration{} + ctx, err := operator.GetInstallationExtension().Prepare(p) + Expect(err).NotTo(HaveOccurred()) + Expect(ctx.NodePrometheusTLS).NotTo(BeNil()) + }) + + It("is a no-op for the Calico variant", func() { + p := newPrep(operatorv1.Calico) + ctx, err := operator.GetInstallationExtension().Prepare(p) + Expect(err).NotTo(HaveOccurred()) + Expect(ctx.NodePrometheusTLS).To(BeNil()) + }) +}) + +func newPrep(variant operatorv1.ProductVariant) operator.InstallationPrep { + scheme := runtime.NewScheme() + Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) + c := ctrlrfake.DefaultFakeClientBuilder(scheme).Build() + + certManager, err := certificatemanager.Create(c, nil, "", common.OperatorNamespace(), certificatemanager.AllowCACreation()) + Expect(err).NotTo(HaveOccurred()) + trustedBundle := certManager.CreateTrustedBundle() + + return operator.InstallationPrep{ + Ctx: context.Background(), + Client: c, + Installation: &operatorv1.InstallationSpec{ + Variant: variant, + }, + FelixConfiguration: &v3.FelixConfiguration{}, + CertificateManager: certManager, + TrustedBundle: trustedBundle, + ClusterDomain: "cluster.local", + } +} diff --git a/pkg/enterprise/register.go b/pkg/enterprise/register.go index d32e8c66b1..a01c9b7cfa 100644 --- a/pkg/enterprise/register.go +++ b/pkg/enterprise/register.go @@ -20,4 +20,5 @@ package enterprise func Register() { registerTypha() registerNode() + registerInstallation() } From 6e426cb3cf2e1ad331f0f265f0f45b695d3e2ed8 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Fri, 29 May 2026 10:04:45 -0700 Subject: [PATCH 13/38] enterprise: render node metrics service via a modifier Moves the calico-node-metrics Service out of OSS node render and into the enterprise node modifier, where it derives ports from ctx.FelixConfiguration. Also exports NodeBGPReporterPort so the modifier can reference it. --- pkg/enterprise/node.go | 74 +++++++++++++++++++++++++++++++++++++ pkg/enterprise/node_test.go | 53 ++++++++++++++++++++++++++ pkg/render/node.go | 58 +---------------------------- pkg/render/node_test.go | 49 +----------------------- pkg/render/render_test.go | 3 +- pkg/render/windows.go | 4 +- 6 files changed, 134 insertions(+), 107 deletions(-) diff --git a/pkg/enterprise/node.go b/pkg/enterprise/node.go index 039303681f..e47c593076 100644 --- a/pkg/enterprise/node.go +++ b/pkg/enterprise/node.go @@ -15,12 +15,24 @@ package enterprise import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + client "sigs.k8s.io/controller-runtime/pkg/client" + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/controller/utils" "github.com/tigera/operator/pkg/operator" "github.com/tigera/operator/pkg/render" ) +const ( + defaultNodeReporterPort = 9081 + defaultFelixMetricsPort = 9091 +) + func registerNode() { operator.OverrideImage(render.ComponentNameNode, func(in *operatorv1.InstallationSpec) (components.Component, bool) { if !in.Variant.IsEnterprise() { @@ -28,4 +40,66 @@ func registerNode() { } return components.ComponentTigeraNode, true }) + operator.Patch(render.ComponentNameNode, patchNode) +} + +func patchNode(ctx operator.Context, objs []client.Object) []client.Object { + if ctx.Installation == nil || !ctx.Installation.Variant.IsEnterprise() { + return objs + } + return append(objs, nodeMetricsService(ctx)) +} + +// nodeMetricsService builds the enterprise-only calico-node-metrics Service. +// Ports are derived from FelixConfiguration exactly as the installation controller does. +func nodeMetricsService(ctx operator.Context) *corev1.Service { + reporterPort := defaultNodeReporterPort + felixPort := defaultFelixMetricsPort + felixEnabled := false + if fc := ctx.FelixConfiguration; fc != nil { + if fc.Spec.PrometheusReporterPort != nil { + reporterPort = *fc.Spec.PrometheusReporterPort + } + if fc.Spec.PrometheusMetricsPort != nil { + felixPort = *fc.Spec.PrometheusMetricsPort + } + felixEnabled = utils.IsFelixPrometheusMetricsEnabled(fc) + } + + ports := []corev1.ServicePort{ + { + Name: "calico-metrics-port", + Port: int32(reporterPort), + TargetPort: intstr.FromInt(reporterPort), + Protocol: corev1.ProtocolTCP, + }, + { + Name: "calico-bgp-metrics-port", + Port: render.NodeBGPReporterPort, + TargetPort: intstr.FromInt(int(render.NodeBGPReporterPort)), + Protocol: corev1.ProtocolTCP, + }, + } + if felixEnabled { + ports = append(ports, corev1.ServicePort{ + Name: "felix-metrics-port", + Port: int32(felixPort), + TargetPort: intstr.FromInt(felixPort), + Protocol: corev1.ProtocolTCP, + }) + } + + return &corev1.Service{ + TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: render.CalicoNodeMetricsService, + Namespace: common.CalicoNamespace, + Labels: map[string]string{"k8s-app": render.CalicoNodeObjectName}, + }, + Spec: corev1.ServiceSpec{ + Selector: map[string]string{"k8s-app": render.CalicoNodeObjectName}, + ClusterIP: "None", + Ports: ports, + }, + } } diff --git a/pkg/enterprise/node_test.go b/pkg/enterprise/node_test.go index d12d9812f2..509fd6b18f 100644 --- a/pkg/enterprise/node_test.go +++ b/pkg/enterprise/node_test.go @@ -18,10 +18,15 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + client "sigs.k8s.io/controller-runtime/pkg/client" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/enterprise" "github.com/tigera/operator/pkg/operator" + "github.com/tigera/operator/pkg/render" ) var _ = Describe("node enterprise image override", func() { @@ -41,3 +46,51 @@ var _ = Describe("node enterprise image override", func() { Expect(operator.ResolveImage("node", components.ComponentCalicoNode, oss)).To(Equal(components.ComponentCalicoNode)) }) }) + +var _ = Describe("node metrics service modifier", func() { + BeforeEach(func() { enterprise.Register() }) + AfterEach(func() { + operator.ResetForTest() + operator.ResetExtensionsForTest() + }) + + It("appends the node metrics service for the enterprise variant", func() { + ctx := operator.Context{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.TigeraSecureEnterprise}} + out := operator.ApplyPatches(render.ComponentNameNode, ctx, []client.Object{}) + svc, ok := operator.FindObject[*corev1.Service](out, render.CalicoNodeMetricsService) + Expect(ok).To(BeTrue()) + // default ports when FelixConfiguration is nil: 9081 + 9900, felix-metrics-port absent + Expect(svc.Spec.Ports).To(HaveLen(2)) + Expect(svc.Spec.Ports[0].Port).To(Equal(int32(9081))) + Expect(svc.Spec.Ports[1].Port).To(Equal(int32(9900))) + }) + + It("derives ports and felix-metrics-port from FelixConfiguration", func() { + reporter := 7081 + metrics := 7091 + enabled := true + ctx := operator.Context{ + Installation: &operatorv1.InstallationSpec{Variant: operatorv1.TigeraSecureEnterprise}, + FelixConfiguration: &v3.FelixConfiguration{Spec: v3.FelixConfigurationSpec{ + PrometheusReporterPort: &reporter, + PrometheusMetricsPort: &metrics, + PrometheusMetricsEnabled: &enabled, + }}, + } + out := operator.ApplyPatches(render.ComponentNameNode, ctx, []client.Object{}) + svc, ok := operator.FindObject[*corev1.Service](out, render.CalicoNodeMetricsService) + Expect(ok).To(BeTrue()) + Expect(svc.Spec.Ports).To(HaveLen(3)) + Expect(svc.Spec.Ports[0].Port).To(Equal(int32(7081))) + Expect(svc.Spec.Ports[2].Name).To(Equal("felix-metrics-port")) + Expect(svc.Spec.Ports[2].Port).To(Equal(int32(7091))) + }) + + It("does not append it for the Calico variant", func() { + ctx := operator.Context{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}} + out := operator.ApplyPatches(render.ComponentNameNode, ctx, []client.Object{}) + _, ok := operator.FindObject[*corev1.Service](out, render.CalicoNodeMetricsService) + Expect(ok).To(BeFalse()) + }) +}) + diff --git a/pkg/render/node.go b/pkg/render/node.go index 9c335d9ea4..2128d55021 100644 --- a/pkg/render/node.go +++ b/pkg/render/node.go @@ -73,9 +73,9 @@ const ( ) var ( - // The port used by calico/node to report Calico Enterprise BGP metrics. + // NodeBGPReporterPort is the port used by calico/node to report Calico Enterprise BGP metrics. // This is currently not intended to be user configurable. - nodeBGPReporterPort int32 = 9900 + NodeBGPReporterPort int32 = 9900 NodeTLSSecretName = "node-certs" NodeTLSSecretNameNonClusterHost = NodeTLSSecretName + TyphaNonClusterHostSuffix @@ -234,11 +234,6 @@ func (c *nodeComponent) Objects() ([]client.Object, []client.Object) { var objsToDelete []client.Object - if c.cfg.Installation.Variant.IsEnterprise() { - // Include Service for exposing node metrics. - objs = append(objs, c.nodeMetricsService()) - } - cniConfig := c.nodeCNIConfigMap() if cniConfig != nil { objs = append(objs, cniConfig) @@ -1777,55 +1772,6 @@ func (c *nodeComponent) nodeLivenessReadinessProbes() (*corev1.Probe, *corev1.Pr return lp, rp } -// nodeMetricsService creates a Service which exposes two endpoints on calico/node for -// reporting Prometheus metrics (for policy enforcement activity and BGP stats). -// This service is used internally by Calico Enterprise and is separate from general -// Prometheus metrics which are user-configurable. -func (c *nodeComponent) nodeMetricsService() *corev1.Service { - ports := []corev1.ServicePort{ - { - Name: "calico-metrics-port", - Port: int32(c.cfg.NodeReporterMetricsPort), - TargetPort: intstr.FromInt(c.cfg.NodeReporterMetricsPort), - Protocol: corev1.ProtocolTCP, - }, - { - Name: "calico-bgp-metrics-port", - Port: nodeBGPReporterPort, - TargetPort: intstr.FromInt(int(nodeBGPReporterPort)), - Protocol: corev1.ProtocolTCP, - }, - } - - if c.cfg.FelixPrometheusMetricsEnabled { - felixMetricsPort := int32(c.cfg.FelixPrometheusMetricsPort) - - ports = append(ports, corev1.ServicePort{ - Name: "felix-metrics-port", - Port: felixMetricsPort, - TargetPort: intstr.FromInt(int(felixMetricsPort)), - Protocol: corev1.ProtocolTCP, - }) - } - - return &corev1.Service{ - TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: CalicoNodeMetricsService, - Namespace: common.CalicoNamespace, - Labels: map[string]string{"k8s-app": CalicoNodeObjectName}, - }, - Spec: corev1.ServiceSpec{ - Selector: map[string]string{"k8s-app": CalicoNodeObjectName}, - // Important: "None" tells Kubernetes that we want a headless service with - // no kube-proxy load balancer. If we omit this then kube-proxy will render - // a huge set of iptables rules for this service since there's an instance - // on every node. - ClusterIP: "None", - Ports: ports, - }, - } -} // getAutodetectionMethod returns the IP auto detection method in a form understandable by the calico/node // startup processing. It returns an empty string if IP auto detection should not be enabled. diff --git a/pkg/render/node_test.go b/pkg/render/node_test.go index e7a4ba37d4..30c772a91d 100644 --- a/pkg/render/node_test.go +++ b/pkg/render/node_test.go @@ -651,7 +651,6 @@ var _ = Describe("Node rendering tests", func() { {name: "calico-cni-plugin", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ServiceAccount"}, {name: "calico-cni-plugin", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, {name: "calico-cni-plugin", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: "calico-node-metrics", ns: "calico-system", group: "", version: "v1", kind: "Service"}, {name: "cni-config", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ConfigMap"}, {name: common.NodeDaemonSetName, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "DaemonSet"}, } @@ -725,10 +724,6 @@ var _ = Describe("Node rendering tests", func() { Expect(ds.Spec.Template.Spec.Containers[0].Env).To(ConsistOf(expectedNodeEnv)) Expect(len(ds.Spec.Template.Spec.Containers[0].Env)).To(Equal(len(expectedNodeEnv))) - // Expect 2 Ports when FelixPrometheusMetricsEnabled is false - ms := rtest.GetResource(resources, "calico-node-metrics", "calico-system", "", "v1", "Service").(*corev1.Service) - Expect(len(ms.Spec.Ports)).To(Equal(2)) - dirMustExist := corev1.HostPathDirectory bpfVol := corev1.Volume{Name: "bpffs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs/bpf", Type: &dirMustExist}}} Expect(ds.Spec.Template.Spec.Volumes).To(ContainElement(bpfVol)) @@ -739,40 +734,6 @@ var _ = Describe("Node rendering tests", func() { verifyProbesAndLifecycle(ds, false, true) }) - It("should render felix service metric with FelixPrometheusMetricPort when FelixPrometheusMetricsEnabled is true", func() { - defaultInstance.Variant = operatorv1.CalicoEnterprise - cfg.NodeReporterMetricsPort = 9081 - cfg.FelixPrometheusMetricsEnabled = true - - component := render.Node(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - - expectedServicePorts := []corev1.ServicePort{ - { - Name: "calico-metrics-port", - Port: int32(cfg.NodeReporterMetricsPort), - TargetPort: intstr.FromInt(cfg.NodeReporterMetricsPort), - Protocol: corev1.ProtocolTCP, - }, - { - Name: "calico-bgp-metrics-port", - Port: 9900, - TargetPort: intstr.FromInt(int(9900)), - Protocol: corev1.ProtocolTCP, - }, - { - Name: "felix-metrics-port", - Port: 9098, - TargetPort: intstr.FromInt(int(9098)), - Protocol: corev1.ProtocolTCP, - }, - } - - // Expect 3 Ports when FelixPrometheusMetricsEnabled is true - ms := rtest.GetResource(resources, "calico-node-metrics", "calico-system", "", "v1", "Service").(*corev1.Service) - Expect(ms.Spec.Ports).To(Equal(expectedServicePorts)) - }) It("should render all resources when using Calico CNI on EKS", func() { expectedResources := []struct { @@ -1639,7 +1600,6 @@ var _ = Describe("Node rendering tests", func() { {name: "calico-cni-plugin", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ServiceAccount"}, {name: "calico-cni-plugin", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, {name: "calico-cni-plugin", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: "calico-node-metrics", ns: "calico-system", group: "", version: "v1", kind: "Service"}, {name: "cni-config", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ConfigMap"}, {name: common.NodeDaemonSetName, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "DaemonSet"}, } @@ -1744,7 +1704,6 @@ var _ = Describe("Node rendering tests", func() { {name: "calico-cni-plugin", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ServiceAccount"}, {name: "calico-cni-plugin", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, {name: "calico-cni-plugin", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: "calico-node-metrics", ns: "calico-system", group: "", version: "v1", kind: "Service"}, {name: "cni-config", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ConfigMap"}, {name: common.NodeDaemonSetName, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "DaemonSet"}, } @@ -1827,10 +1786,6 @@ var _ = Describe("Node rendering tests", func() { Expect(len(ds.Spec.Template.Spec.Containers[0].Env)).To(Equal(len(expectedNodeEnv))) verifyProbesAndLifecycle(ds, true, true) - - // The metrics service should have the correct configuration. - ms := rtest.GetResource(resources, "calico-node-metrics", "calico-system", "", "v1", "Service").(*corev1.Service) - Expect(ms.Spec.ClusterIP).To(Equal("None"), "metrics service should be headless to prevent kube-proxy from rendering too many iptables rules") }) It("should render volumes and node volumemounts when bird templates are provided", func() { @@ -2094,7 +2049,7 @@ var _ = Describe("Node rendering tests", func() { component := render.Node(&cfg) Expect(component.ResolveImages(nil)).To(BeNil()) resources, _ := component.Objects() - Expect(len(resources)).To(Equal(defaultNumExpectedResources + 1)) + Expect(len(resources)).To(Equal(defaultNumExpectedResources)) dsResource := rtest.GetResource(resources, "calico-node", "calico-system", "apps", "v1", "DaemonSet") Expect(dsResource).ToNot(BeNil()) @@ -2115,7 +2070,7 @@ var _ = Describe("Node rendering tests", func() { component := render.Node(&cfg) Expect(component.ResolveImages(nil)).To(BeNil()) resources, _ := component.Objects() - Expect(len(resources)).To(Equal(defaultNumExpectedResources + 1)) + Expect(len(resources)).To(Equal(defaultNumExpectedResources)) dsResource := rtest.GetResource(resources, "calico-node", "calico-system", "apps", "v1", "DaemonSet") Expect(dsResource).ToNot(BeNil()) diff --git a/pkg/render/render_test.go b/pkg/render/render_test.go index 2b97cfaeb9..42afa04570 100644 --- a/pkg/render/render_test.go +++ b/pkg/render/render_test.go @@ -234,7 +234,7 @@ var _ = Describe("Rendering tests", func() { instance.NodeMetricsPort = &nodeMetricsPort c, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 9094, 0, nil, nil) Expect(err).To(BeNil(), "Expected Calico to create successfully %s", err) - Expect(componentCount(c)).To(Equal((5 + 3 + 4 + 1 + 6 + 6 + 1 + 2) + 1 + 1)) + Expect(componentCount(c)).To(Equal((5 + 3 + 4 + 1 + 6 + 6 + 1 + 2) + 1)) }) It("should render all resources when variant is Tigera Secure and Management Cluster", func() { @@ -267,7 +267,6 @@ var _ = Describe("Rendering tests", func() { &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "calico-cni-plugin", Namespace: common.CalicoNamespace}, TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}}, &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-cni-plugin"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-cni-plugin"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "calico-node-metrics", Namespace: common.CalicoNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}}, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cni-config", Namespace: common.CalicoNamespace}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, &appsv1.DaemonSet{ObjectMeta: metav1.ObjectMeta{Name: common.NodeDaemonSetName, Namespace: common.CalicoNamespace}, TypeMeta: metav1.TypeMeta{Kind: "DaemonSet", APIVersion: "apps/v1"}}, diff --git a/pkg/render/windows.go b/pkg/render/windows.go index d642178133..a796bd5d0d 100644 --- a/pkg/render/windows.go +++ b/pkg/render/windows.go @@ -163,8 +163,8 @@ func (c *windowsComponent) nodeMetricsService() *corev1.Service { }, { Name: "calico-bgp-metrics-port", - Port: nodeBGPReporterPort, - TargetPort: intstr.FromInt(int(nodeBGPReporterPort)), + Port: NodeBGPReporterPort, + TargetPort: intstr.FromInt(int(NodeBGPReporterPort)), Protocol: corev1.ProtocolTCP, }, }, From c74b106d2dc10b2da8685658bb176bfad50d3b2f Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Fri, 29 May 2026 13:56:08 -0700 Subject: [PATCH 14/38] variant-extensions: gofmt and replace deprecated TigeraSecureEnterprise in tests --- pkg/controller/installation/core_controller.go | 4 ++-- pkg/controller/utils/component_test.go | 12 +++++++----- pkg/enterprise/installation.go | 2 +- pkg/enterprise/installation_test.go | 4 ++-- pkg/enterprise/node.go | 4 ++-- pkg/enterprise/node_test.go | 7 +++---- pkg/enterprise/typha_test.go | 2 +- pkg/operator/image_test.go | 2 +- pkg/render/node.go | 1 - pkg/render/node_test.go | 1 - 10 files changed, 19 insertions(+), 20 deletions(-) diff --git a/pkg/controller/installation/core_controller.go b/pkg/controller/installation/core_controller.go index bf92bf3e2f..af2fcd6735 100644 --- a/pkg/controller/installation/core_controller.go +++ b/pkg/controller/installation/core_controller.go @@ -63,7 +63,6 @@ import ( "github.com/tigera/operator/pkg/active" "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/components" - "github.com/tigera/operator/pkg/operator" "github.com/tigera/operator/pkg/controller/certificatemanager" "github.com/tigera/operator/pkg/controller/ippool" "github.com/tigera/operator/pkg/controller/k8sapi" @@ -78,9 +77,10 @@ import ( "github.com/tigera/operator/pkg/dns" "github.com/tigera/operator/pkg/imports/admission" "github.com/tigera/operator/pkg/imports/crds" + "github.com/tigera/operator/pkg/operator" "github.com/tigera/operator/pkg/render" rcertificatemanagement "github.com/tigera/operator/pkg/render/certificatemanagement" -"github.com/tigera/operator/pkg/render/common/networkpolicy" + "github.com/tigera/operator/pkg/render/common/networkpolicy" "github.com/tigera/operator/pkg/render/common/resourcequota" "github.com/tigera/operator/pkg/render/goldmane" "github.com/tigera/operator/pkg/render/kubecontrollers" diff --git a/pkg/controller/utils/component_test.go b/pkg/controller/utils/component_test.go index d4b34cb871..88da64d06d 100644 --- a/pkg/controller/utils/component_test.go +++ b/pkg/controller/utils/component_test.go @@ -2495,8 +2495,10 @@ type namedFakeComponent struct { obj client.Object } -func (f *namedFakeComponent) Name() string { return f.name } -func (f *namedFakeComponent) ResolveImages(*operatorv1.ImageSet) error { return nil } -func (f *namedFakeComponent) Objects() ([]client.Object, []client.Object) { return []client.Object{f.obj}, nil } -func (f *namedFakeComponent) Ready() bool { return true } -func (f *namedFakeComponent) SupportedOSType() rmeta.OSType { return rmeta.OSTypeLinux } +func (f *namedFakeComponent) Name() string { return f.name } +func (f *namedFakeComponent) ResolveImages(*operatorv1.ImageSet) error { return nil } +func (f *namedFakeComponent) Objects() ([]client.Object, []client.Object) { + return []client.Object{f.obj}, nil +} +func (f *namedFakeComponent) Ready() bool { return true } +func (f *namedFakeComponent) SupportedOSType() rmeta.OSType { return rmeta.OSTypeLinux } diff --git a/pkg/enterprise/installation.go b/pkg/enterprise/installation.go index 7b0abaf4b5..dc99e70eaf 100644 --- a/pkg/enterprise/installation.go +++ b/pkg/enterprise/installation.go @@ -20,10 +20,10 @@ import ( "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/dns" - "github.com/tigera/operator/pkg/render/monitor" "github.com/tigera/operator/pkg/operator" "github.com/tigera/operator/pkg/render" relasticsearch "github.com/tigera/operator/pkg/render/common/elasticsearch" + "github.com/tigera/operator/pkg/render/monitor" ) type installationExtension struct{} diff --git a/pkg/enterprise/installation_test.go b/pkg/enterprise/installation_test.go index dffe6a7470..bf64a2ad7d 100644 --- a/pkg/enterprise/installation_test.go +++ b/pkg/enterprise/installation_test.go @@ -40,14 +40,14 @@ var _ = Describe("installation enterprise extension", func() { It("rejects a zero prometheus reporter port", func() { port := 0 - p := newPrep(operatorv1.TigeraSecureEnterprise) + p := newPrep(operatorv1.CalicoEnterprise) p.FelixConfiguration = &v3.FelixConfiguration{Spec: v3.FelixConfigurationSpec{PrometheusReporterPort: &port}} _, err := operator.GetInstallationExtension().Prepare(p) Expect(err).To(HaveOccurred()) }) It("creates the node prometheus keypair for the enterprise variant", func() { - p := newPrep(operatorv1.TigeraSecureEnterprise) + p := newPrep(operatorv1.CalicoEnterprise) p.FelixConfiguration = &v3.FelixConfiguration{} ctx, err := operator.GetInstallationExtension().Prepare(p) Expect(err).NotTo(HaveOccurred()) diff --git a/pkg/enterprise/node.go b/pkg/enterprise/node.go index e47c593076..f1ea2bcce2 100644 --- a/pkg/enterprise/node.go +++ b/pkg/enterprise/node.go @@ -29,8 +29,8 @@ import ( ) const ( - defaultNodeReporterPort = 9081 - defaultFelixMetricsPort = 9091 + defaultNodeReporterPort = 9081 + defaultFelixMetricsPort = 9091 ) func registerNode() { diff --git a/pkg/enterprise/node_test.go b/pkg/enterprise/node_test.go index 509fd6b18f..d2370fda6e 100644 --- a/pkg/enterprise/node_test.go +++ b/pkg/enterprise/node_test.go @@ -37,7 +37,7 @@ var _ = Describe("node enterprise image override", func() { }) It("selects the enterprise node image for the enterprise variant", func() { - ent := &operatorv1.InstallationSpec{Variant: operatorv1.TigeraSecureEnterprise} + ent := &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise} Expect(operator.ResolveImage("node", components.ComponentCalicoNode, ent)).To(Equal(components.ComponentTigeraNode)) }) @@ -55,7 +55,7 @@ var _ = Describe("node metrics service modifier", func() { }) It("appends the node metrics service for the enterprise variant", func() { - ctx := operator.Context{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.TigeraSecureEnterprise}} + ctx := operator.Context{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}} out := operator.ApplyPatches(render.ComponentNameNode, ctx, []client.Object{}) svc, ok := operator.FindObject[*corev1.Service](out, render.CalicoNodeMetricsService) Expect(ok).To(BeTrue()) @@ -70,7 +70,7 @@ var _ = Describe("node metrics service modifier", func() { metrics := 7091 enabled := true ctx := operator.Context{ - Installation: &operatorv1.InstallationSpec{Variant: operatorv1.TigeraSecureEnterprise}, + Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}, FelixConfiguration: &v3.FelixConfiguration{Spec: v3.FelixConfigurationSpec{ PrometheusReporterPort: &reporter, PrometheusMetricsPort: &metrics, @@ -93,4 +93,3 @@ var _ = Describe("node metrics service modifier", func() { Expect(ok).To(BeFalse()) }) }) - diff --git a/pkg/enterprise/typha_test.go b/pkg/enterprise/typha_test.go index 67330b7539..9b41d448fa 100644 --- a/pkg/enterprise/typha_test.go +++ b/pkg/enterprise/typha_test.go @@ -52,7 +52,7 @@ var _ = Describe("typha enterprise modifier", func() { It("adds enterprise RBAC and MULTI_INTERFACE_MODE for the enterprise variant", func() { ctx := operator.Context{Installation: &operatorv1.InstallationSpec{ - Variant: operatorv1.TigeraSecureEnterprise, + Variant: operatorv1.CalicoEnterprise, CalicoNetwork: &operatorv1.CalicoNetworkSpec{MultiInterfaceMode: &multiMode}, }} out := operator.ApplyPatches(render.ComponentNameTypha, ctx, newObjs()) diff --git a/pkg/operator/image_test.go b/pkg/operator/image_test.go index 93e9beb67d..b972665e17 100644 --- a/pkg/operator/image_test.go +++ b/pkg/operator/image_test.go @@ -36,7 +36,7 @@ var _ = Describe("image overrides", func() { return components.ComponentTigeraNode, true }) - ent := &operatorv1.InstallationSpec{Variant: operatorv1.TigeraSecureEnterprise} + ent := &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise} Expect(operator.ResolveImage("node", components.ComponentCalicoNode, ent)).To(Equal(components.ComponentTigeraNode)) }) diff --git a/pkg/render/node.go b/pkg/render/node.go index 2128d55021..b93775d197 100644 --- a/pkg/render/node.go +++ b/pkg/render/node.go @@ -1772,7 +1772,6 @@ func (c *nodeComponent) nodeLivenessReadinessProbes() (*corev1.Probe, *corev1.Pr return lp, rp } - // getAutodetectionMethod returns the IP auto detection method in a form understandable by the calico/node // startup processing. It returns an empty string if IP auto detection should not be enabled. func getAutodetectionMethod(ad *operatorv1.NodeAddressAutodetection) string { diff --git a/pkg/render/node_test.go b/pkg/render/node_test.go index 30c772a91d..ed02631b6c 100644 --- a/pkg/render/node_test.go +++ b/pkg/render/node_test.go @@ -734,7 +734,6 @@ var _ = Describe("Node rendering tests", func() { verifyProbesAndLifecycle(ds, false, true) }) - It("should render all resources when using Calico CNI on EKS", func() { expectedResources := []struct { name string From fd4e8355b358f25f685b5dfcc82b0c40c5842f23 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Wed, 3 Jun 2026 15:31:46 -0700 Subject: [PATCH 15/38] Generalize the variant extension mechanism The registry package is renamed to extensions. The installation controller builds the render context through a registered factory, and the componentHandler applies registered modifiers to component output. The node and typha variant branches now live in enterprise modifiers, and the calico log directory is mounted for both variants. --- cmd/main.go | 6 +- .../installation/core_controller.go | 110 ++--- pkg/controller/utils/component.go | 14 +- .../utils/component_enterprise_test.go | 87 ++++ pkg/controller/utils/component_test.go | 10 +- pkg/enterprise/installation.go | 61 +-- pkg/enterprise/installation_test.go | 51 ++- pkg/enterprise/node.go | 185 ++++++++- pkg/enterprise/node_test.go | 166 ++++++-- pkg/enterprise/typha.go | 10 +- pkg/enterprise/typha_test.go | 15 +- .../extensions_suite_test.go} | 6 +- pkg/extensions/factory.go | 127 ++++++ pkg/extensions/factory_test.go | 73 ++++ pkg/{operator => extensions}/image.go | 2 +- pkg/{operator => extensions}/image_test.go | 14 +- .../patch.go => extensions/modifier.go} | 34 +- .../modifier_test.go} | 34 +- .../rendercontext.go} | 16 +- pkg/operator/extension.go | 59 --- pkg/operator/extension_test.go | 42 -- pkg/render/component.go | 21 +- pkg/render/enterprise_setup_test.go | 13 +- pkg/render/node.go | 123 +----- pkg/render/node_enterprise_test.go | 172 ++++++++ pkg/render/node_test.go | 375 ++---------------- pkg/render/render_test.go | 21 +- 27 files changed, 1037 insertions(+), 810 deletions(-) create mode 100644 pkg/controller/utils/component_enterprise_test.go rename pkg/{operator/operator_suite_test.go => extensions/extensions_suite_test.go} (88%) create mode 100644 pkg/extensions/factory.go create mode 100644 pkg/extensions/factory_test.go rename pkg/{operator => extensions}/image.go (98%) rename pkg/{operator => extensions}/image_test.go (69%) rename pkg/{operator/patch.go => extensions/modifier.go} (50%) rename pkg/{operator/patch_test.go => extensions/modifier_test.go} (53%) rename pkg/{operator/context.go => extensions/rendercontext.go} (70%) delete mode 100644 pkg/operator/extension.go delete mode 100644 pkg/operator/extension_test.go create mode 100644 pkg/render/node_enterprise_test.go diff --git a/cmd/main.go b/cmd/main.go index b086ca605a..0613e68612 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -529,9 +529,9 @@ If a value other than 'all' is specified, the first CRD with a prefix of the spe os.Exit(1) } - // Wire in-repo enterprise modifiers and extensions into the operator - // registries. After the monorepo split this call moves to calico-private's - // main. + // Wire the in-repo Calico Enterprise extensions (the render context factory, + // modifiers, and image overrides) into the operator registries. After the + // monorepo split this call moves to calico-private's main. enterprise.Register() err = controller.AddToManager(mgr, options) diff --git a/pkg/controller/installation/core_controller.go b/pkg/controller/installation/core_controller.go index d1dfceeecb..bbeb8582af 100644 --- a/pkg/controller/installation/core_controller.go +++ b/pkg/controller/installation/core_controller.go @@ -75,9 +75,9 @@ import ( "github.com/tigera/operator/pkg/controller/utils/imageset" "github.com/tigera/operator/pkg/ctrlruntime" "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/imports/admission" "github.com/tigera/operator/pkg/imports/crds" - "github.com/tigera/operator/pkg/operator" "github.com/tigera/operator/pkg/render" rcertificatemanagement "github.com/tigera/operator/pkg/render/certificatemanagement" "github.com/tigera/operator/pkg/render/common/networkpolicy" @@ -91,11 +91,11 @@ import ( const ( techPreviewFeatureSeccompApparmor = "tech-preview.operator.tigera.io/node-apparmor-profile" - // The default port used by calico/node to report Calico Enterprise internal metrics. - // This is separate from the calico/node prometheus metrics port, which is user configurable. + // defaultNodeReporterPort is the default port calico/node uses to report Calico + // Enterprise internal metrics. The Linux node path derives this in the + // enterprise node modifier; this copy serves the Windows controller, which + // still carries its enterprise logic inline. defaultNodeReporterPort = 9081 - - defaultFelixMetricsDefaultPort = 9091 ) const InstallationName string = "calico" @@ -1208,47 +1208,28 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile return reconcile.Result{}, err } - // nodeReporterMetricsPort is a port used in Enterprise to host internal metrics. - // Operator is responsible for creating a service which maps to that port. - nodeReporterMetricsPort := defaultNodeReporterPort - felixPrometheusMetricsPort := defaultFelixMetricsDefaultPort calicoVersion := components.CalicoRelease if instance.Spec.Variant.IsEnterprise() { calicoVersion = components.EnterpriseRelease - if felixConfiguration.Spec.PrometheusReporterPort != nil { - nodeReporterMetricsPort = *felixConfiguration.Spec.PrometheusReporterPort - } - if felixConfiguration.Spec.PrometheusMetricsPort != nil { - felixPrometheusMetricsPort = *felixConfiguration.Spec.PrometheusMetricsPort - } - } - - // Run the registered installation extension. For the enterprise variant this - // validates config and creates the node-prometheus certificate, adding it (and - // the prometheus/esgw certs) to the trusted bundle. Returns the render Context - // consumed by registered modifiers. - var modCtx operator.Context - if ext := operator.GetInstallationExtension(); ext != nil { - modCtx, err = ext.Prepare(operator.InstallationPrep{ - Ctx: ctx, - Client: r.client, - Installation: &instance.Spec, - FelixConfiguration: felixConfiguration, - CertificateManager: certificateManager, - TrustedBundle: typhaNodeTLS.TrustedBundle, - ClusterDomain: r.clusterDomain, - }) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceCreateError, "Error preparing enterprise installation", err, reqLogger) - return reconcile.Result{}, err - } - } else { - modCtx = operator.Context{ - Installation: &instance.Spec, - FelixConfiguration: felixConfiguration, - ClusterDomain: r.clusterDomain, - TrustedBundle: typhaNodeTLS.TrustedBundle, - } + } + + // Build the render context handed to registered modifiers. The core operator + // factory returns just the base context; an extension's factory additionally + // does controller-side work for its variant - validating config and creating + // the node-prometheus certificate, adding it (and the prometheus/esgw certs) + // to the trusted bundle - and may abort the reconcile by returning an error. + modCtx, err := extensions.GetRenderContextFactory().New( + extensions.WithContext(ctx), + extensions.WithClient(r.client), + extensions.WithInstallation(&instance.Spec), + extensions.WithFelixConfiguration(felixConfiguration), + extensions.WithCertificateManager(certificateManager), + extensions.WithTrustedBundle(typhaNodeTLS.TrustedBundle), + extensions.WithClusterDomain(r.clusterDomain), + ) + if err != nil { + r.status.SetDegraded(operatorv1.ResourceCreateError, "Error preparing installation extension", err, reqLogger) + return reconcile.Result{}, err } nodePrometheusTLS := modCtx.NodePrometheusTLS @@ -1289,7 +1270,7 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile } // Create a component handler to create or update the rendered components. - handler := r.newComponentHandler(log, r.client, r.scheme, instance, utils.WithContext(modCtx)) + handler := r.newComponentHandler(log, r.client, r.scheme, instance, utils.WithRenderContext(modCtx)) // Render namespaces first - this ensures that any other controllers blocked on namespace existence can proceed. namespaceCfg := &render.NamespaceConfiguration{ @@ -1525,28 +1506,25 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile // Build a configuration for rendering calico/node. nodeCfg := render.NodeConfiguration{ - GoldmaneRunning: goldmaneRunning, - K8sServiceEp: k8sapi.Endpoint, - Installation: &instance.Spec, - IPPools: crdPoolsToOperator(currentPools.Items), - LogCollector: logCollector, - BirdTemplates: birdTemplates, - TLS: typhaNodeTLS, - ClusterDomain: r.clusterDomain, - DefaultDNSPolicy: defaultDNSPolicy, - DefaultDNSConfig: defaultDNSConfig, - GoldmaneIP: goldmaneIP, - NodeReporterMetricsPort: nodeReporterMetricsPort, - BGPLayouts: bgpLayout, - NodeAppArmorProfile: nodeAppArmorProfile, - MigrateNamespaces: needsNamespaceMigration, - CanRemoveCNIFinalizer: canRemoveCNI, - PrometheusServerTLS: nodePrometheusTLS, - FelixHealthPort: *felixConfiguration.Spec.HealthPort, - NodeCgroupV2Path: felixConfiguration.Spec.CgroupV2Path, - FelixPrometheusMetricsEnabled: utils.IsFelixPrometheusMetricsEnabled(felixConfiguration), - FelixPrometheusMetricsPort: felixPrometheusMetricsPort, - V3CRDs: r.v3CRDs, + GoldmaneRunning: goldmaneRunning, + K8sServiceEp: k8sapi.Endpoint, + Installation: &instance.Spec, + IPPools: crdPoolsToOperator(currentPools.Items), + LogCollector: logCollector, + BirdTemplates: birdTemplates, + TLS: typhaNodeTLS, + ClusterDomain: r.clusterDomain, + DefaultDNSPolicy: defaultDNSPolicy, + DefaultDNSConfig: defaultDNSConfig, + GoldmaneIP: goldmaneIP, + BGPLayouts: bgpLayout, + NodeAppArmorProfile: nodeAppArmorProfile, + MigrateNamespaces: needsNamespaceMigration, + CanRemoveCNIFinalizer: canRemoveCNI, + PrometheusServerTLS: nodePrometheusTLS, + FelixHealthPort: *felixConfiguration.Spec.HealthPort, + NodeCgroupV2Path: felixConfiguration.Spec.CgroupV2Path, + V3CRDs: r.v3CRDs, } if bgpConfiguration.Spec.BindMode != nil { diff --git a/pkg/controller/utils/component.go b/pkg/controller/utils/component.go index 7d63736fcd..3e343c4540 100644 --- a/pkg/controller/utils/component.go +++ b/pkg/controller/utils/component.go @@ -47,7 +47,7 @@ import ( "github.com/tigera/operator/pkg/apigroup" "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/controller/status" - "github.com/tigera/operator/pkg/operator" + "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" rmeta "github.com/tigera/operator/pkg/render/common/meta" ) @@ -78,8 +78,9 @@ type ComponentHandler interface { // ComponentHandlerOption configures a componentHandler. type ComponentHandlerOption func(*componentHandler) -// WithContext supplies the operator.Context passed to registered render patches. -func WithContext(ctx operator.Context) ComponentHandlerOption { +// WithRenderContext supplies the extensions.RenderContext passed to registered +// render modifiers. +func WithRenderContext(ctx extensions.RenderContext) ComponentHandlerOption { return func(c *componentHandler) { c.modCtx = ctx } } @@ -106,7 +107,7 @@ type componentHandler struct { log logr.Logger createOnly bool apiGroupEnvs []v1.EnvVar - modCtx operator.Context + modCtx extensions.RenderContext } func (c *componentHandler) SetCreateOnly() { @@ -467,8 +468,8 @@ func (c *componentHandler) CreateOrUpdateOrDelete(ctx context.Context, component var cronJobs []types.NamespacedName objsToCreate, objsToDelete := component.Objects() - if named, ok := component.(render.Named); ok { - objsToCreate = operator.ApplyPatches(named.Name(), c.modCtx, objsToCreate) + if named, ok := component.(render.Extensible); ok { + objsToCreate = extensions.ApplyModifiers(named.Name(), c.modCtx, objsToCreate) } // Load the InstallationSpec once and reuse it for every object: createOrUpdateObject needs it @@ -1150,7 +1151,6 @@ func addComponentLabel(obj metav1.Object, cr metav1.Object) { owner, ok := cr.(runtime.Object) if ok && owner.GetObjectKind() != nil && owner.GetObjectKind() != nil { obj.GetLabels()["app.kubernetes.io/component"] = sanitizeLabel(owner.GetObjectKind().GroupVersionKind().GroupKind().String()) - } } } diff --git a/pkg/controller/utils/component_enterprise_test.go b/pkg/controller/utils/component_enterprise_test.go new file mode 100644 index 0000000000..a59e1394b6 --- /dev/null +++ b/pkg/controller/utils/component_enterprise_test.go @@ -0,0 +1,87 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils_test + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + logf "sigs.k8s.io/controller-runtime/pkg/log" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/apis" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/controller/k8sapi" + "github.com/tigera/operator/pkg/controller/utils" + ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/enterprise" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/render" +) + +// This exercises the full path comment-by-comment: a real render component goes +// through CreateOrUpdateOrDelete with an enterprise RenderContext, and the +// registered modifier must match the real render output by name. If render ever +// renames the typha ClusterRole, the modifier silently no-ops and this fails. +var _ = Describe("componentHandler enterprise modifier integration", func() { + BeforeEach(func() { enterprise.Register() }) + AfterEach(func() { extensions.ResetForTest() }) + + It("applies the enterprise typha modifier to real render output", func() { + scheme := runtime.NewScheme() + Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) + cli := ctrlrfake.DefaultFakeClientBuilder(scheme).Build() + + certManager, err := certificatemanager.Create(cli, nil, "", common.OperatorNamespace(), certificatemanager.AllowCACreation()) + Expect(err).NotTo(HaveOccurred()) + nodeKeyPair, err := certManager.GetOrCreateKeyPair(cli, render.NodeTLSSecretName, common.OperatorNamespace(), []string{render.FelixCommonName}) + Expect(err).NotTo(HaveOccurred()) + typhaKeyPair, err := certManager.GetOrCreateKeyPair(cli, render.TyphaTLSSecretName, common.OperatorNamespace(), []string{render.TyphaCommonName}) + Expect(err).NotTo(HaveOccurred()) + + instance := &operatorv1.InstallationSpec{ + Variant: operatorv1.CalicoEnterprise, + CNI: &operatorv1.CNISpec{Type: operatorv1.PluginCalico}, + } + comp := render.Typha(&render.TyphaConfiguration{ + K8sServiceEp: k8sapi.ServiceEndpoint{}, + Installation: instance, + ClusterDomain: dns.DefaultClusterDomain, + FelixHealthPort: 9099, + TLS: &render.TyphaNodeTLS{ + TrustedBundle: certManager.CreateTrustedBundle(), + TyphaSecret: typhaKeyPair, + TyphaCommonName: render.TyphaCommonName, + NodeSecret: nodeKeyPair, + NodeCommonName: render.FelixCommonName, + }, + }) + + renderCtx := extensions.RenderContext{Installation: instance} + handler := utils.NewComponentHandler(logf.Log, cli, scheme, nil, utils.WithRenderContext(renderCtx)) + Expect(handler.CreateOrUpdateOrDelete(context.Background(), comp, nil)).NotTo(HaveOccurred()) + + role := &rbacv1.ClusterRole{} + Expect(cli.Get(context.Background(), client.ObjectKey{Name: "calico-typha"}, role)).NotTo(HaveOccurred()) + Expect(role.Rules).To(ContainElement(HaveField("Resources", ContainElement("licensekeys")))) + }) +}) diff --git a/pkg/controller/utils/component_test.go b/pkg/controller/utils/component_test.go index 1aa5d4be63..3081adfe19 100644 --- a/pkg/controller/utils/component_test.go +++ b/pkg/controller/utils/component_test.go @@ -46,7 +46,7 @@ import ( "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/controller/status" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" - "github.com/tigera/operator/pkg/operator" + "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" rmeta "github.com/tigera/operator/pkg/render/common/meta" ) @@ -2487,13 +2487,13 @@ func (mc *mockClient) SubResource(subResource string) client.SubResourceClient { panic("SubResource not implemented in mockClient") } -var _ = Describe("componentHandler patch application", func() { +var _ = Describe("componentHandler modifier application", func() { AfterEach(func() { - operator.ResetForTest() + extensions.ResetForTest() }) - It("applies registered patches to a named component before create", func() { - operator.Patch("fake", func(ctx operator.Context, objs []client.Object) []client.Object { + It("applies registered modifiers to a named component before create", func() { + extensions.Modify("fake", func(ctx extensions.RenderContext, objs []client.Object) []client.Object { cm := objs[0].(*corev1.ConfigMap) cm.Data = map[string]string{"patched": "yes"} return objs diff --git a/pkg/enterprise/installation.go b/pkg/enterprise/installation.go index dc99e70eaf..596229a635 100644 --- a/pkg/enterprise/installation.go +++ b/pkg/enterprise/installation.go @@ -20,61 +20,64 @@ import ( "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/dns" - "github.com/tigera/operator/pkg/operator" + "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" relasticsearch "github.com/tigera/operator/pkg/render/common/elasticsearch" "github.com/tigera/operator/pkg/render/monitor" ) -type installationExtension struct{} +// installationFactory is the Calico Enterprise RenderContextFactory. It builds +// the base render context and then does the controller-side work the modifiers +// can't: validating config and creating/fetching the certificates that feed the +// trusted bundle. +type installationFactory struct{} func registerInstallation() { - operator.RegisterInstallationExtension(&installationExtension{}) + extensions.RegisterRenderContextFactory(&installationFactory{}) } -func (e *installationExtension) Prepare(p operator.InstallationPrep) (operator.Context, error) { - ctx := operator.Context{ - Installation: p.Installation, - FelixConfiguration: p.FelixConfiguration, - ClusterDomain: p.ClusterDomain, - TrustedBundle: p.TrustedBundle, - } - if !p.Installation.Variant.IsEnterprise() { - return ctx, nil +func (f *installationFactory) New(opts ...extensions.RenderContextOption) (extensions.RenderContext, error) { + in := extensions.ApplyInputs(opts...) + rc := extensions.BaseRenderContext(in) + if in.Installation == nil || !in.Installation.Variant.IsEnterprise() { + return rc, nil } - // Reject the unsupported zero reporter port. (Port value derivation stays in - // the OSS controller; only validation moves here.) - if p.FelixConfiguration.Spec.PrometheusReporterPort != nil && *p.FelixConfiguration.Spec.PrometheusReporterPort == 0 { - return ctx, errors.New("felixConfiguration prometheusReporterPort=0 not supported") + // Reject the unsupported zero reporter port. The port value itself is derived + // in the node modifier; only this validation lives here. + if in.FelixConfiguration.Spec.PrometheusReporterPort != nil && *in.FelixConfiguration.Spec.PrometheusReporterPort == 0 { + return rc, errors.New("felixConfiguration prometheusReporterPort=0 not supported") } - nodePrometheusTLS, err := p.CertificateManager.GetOrCreateKeyPair( - p.Client, render.NodePrometheusTLSServerSecret, common.OperatorNamespace(), - dns.GetServiceDNSNames(render.CalicoNodeMetricsService, common.CalicoNamespace, p.ClusterDomain)) + nodePrometheusTLS, err := in.CertificateManager.GetOrCreateKeyPair( + in.Client, + render.NodePrometheusTLSServerSecret, + common.OperatorNamespace(), + dns.GetServiceDNSNames(render.CalicoNodeMetricsService, common.CalicoNamespace, in.ClusterDomain), + ) if err != nil { - return ctx, fmt.Errorf("error creating node prometheus TLS certificate: %w", err) + return rc, fmt.Errorf("error creating node prometheus TLS certificate: %w", err) } if nodePrometheusTLS != nil { - p.TrustedBundle.AddCertificates(nodePrometheusTLS) + in.TrustedBundle.AddCertificates(nodePrometheusTLS) } - ctx.NodePrometheusTLS = nodePrometheusTLS + rc.NodePrometheusTLS = nodePrometheusTLS - prometheusClientCert, err := p.CertificateManager.GetCertificate(p.Client, monitor.PrometheusClientTLSSecretName, common.OperatorNamespace()) + prometheusClientCert, err := in.CertificateManager.GetCertificate(in.Client, monitor.PrometheusClientTLSSecretName, common.OperatorNamespace()) if err != nil { - return ctx, fmt.Errorf("unable to fetch prometheus certificate: %w", err) + return rc, fmt.Errorf("unable to fetch prometheus certificate: %w", err) } if prometheusClientCert != nil { - p.TrustedBundle.AddCertificates(prometheusClientCert) + in.TrustedBundle.AddCertificates(prometheusClientCert) } - esgwCertificate, err := p.CertificateManager.GetCertificate(p.Client, relasticsearch.PublicCertSecret, common.OperatorNamespace()) + esgwCertificate, err := in.CertificateManager.GetCertificate(in.Client, relasticsearch.PublicCertSecret, common.OperatorNamespace()) if err != nil { - return ctx, fmt.Errorf("failed to retrieve / validate %s: %w", relasticsearch.PublicCertSecret, err) + return rc, fmt.Errorf("failed to retrieve / validate %s: %w", relasticsearch.PublicCertSecret, err) } if esgwCertificate != nil { - p.TrustedBundle.AddCertificates(esgwCertificate) + in.TrustedBundle.AddCertificates(esgwCertificate) } - return ctx, nil + return rc, nil } diff --git a/pkg/enterprise/installation_test.go b/pkg/enterprise/installation_test.go index bf64a2ad7d..18c38c312a 100644 --- a/pkg/enterprise/installation_test.go +++ b/pkg/enterprise/installation_test.go @@ -21,48 +21,45 @@ import ( . "github.com/onsi/gomega" v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + "k8s.io/apimachinery/pkg/runtime" + operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/apis" "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/controller/certificatemanager" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" "github.com/tigera/operator/pkg/enterprise" - "github.com/tigera/operator/pkg/operator" - "k8s.io/apimachinery/pkg/runtime" + "github.com/tigera/operator/pkg/extensions" ) -var _ = Describe("installation enterprise extension", func() { +var _ = Describe("installation render context factory", func() { BeforeEach(func() { enterprise.Register() }) - AfterEach(func() { - operator.ResetForTest() - operator.ResetExtensionsForTest() - }) + AfterEach(func() { extensions.ResetForTest() }) It("rejects a zero prometheus reporter port", func() { port := 0 - p := newPrep(operatorv1.CalicoEnterprise) - p.FelixConfiguration = &v3.FelixConfiguration{Spec: v3.FelixConfigurationSpec{PrometheusReporterPort: &port}} - _, err := operator.GetInstallationExtension().Prepare(p) + opts := newOpts(operatorv1.CalicoEnterprise) + opts = append(opts, extensions.WithFelixConfiguration(&v3.FelixConfiguration{ + Spec: v3.FelixConfigurationSpec{PrometheusReporterPort: &port}, + })) + _, err := extensions.GetRenderContextFactory().New(opts...) Expect(err).To(HaveOccurred()) }) It("creates the node prometheus keypair for the enterprise variant", func() { - p := newPrep(operatorv1.CalicoEnterprise) - p.FelixConfiguration = &v3.FelixConfiguration{} - ctx, err := operator.GetInstallationExtension().Prepare(p) + rc, err := extensions.GetRenderContextFactory().New(newOpts(operatorv1.CalicoEnterprise)...) Expect(err).NotTo(HaveOccurred()) - Expect(ctx.NodePrometheusTLS).NotTo(BeNil()) + Expect(rc.NodePrometheusTLS).NotTo(BeNil()) }) It("is a no-op for the Calico variant", func() { - p := newPrep(operatorv1.Calico) - ctx, err := operator.GetInstallationExtension().Prepare(p) + rc, err := extensions.GetRenderContextFactory().New(newOpts(operatorv1.Calico)...) Expect(err).NotTo(HaveOccurred()) - Expect(ctx.NodePrometheusTLS).To(BeNil()) + Expect(rc.NodePrometheusTLS).To(BeNil()) }) }) -func newPrep(variant operatorv1.ProductVariant) operator.InstallationPrep { +func newOpts(variant operatorv1.ProductVariant) []extensions.RenderContextOption { scheme := runtime.NewScheme() Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) c := ctrlrfake.DefaultFakeClientBuilder(scheme).Build() @@ -71,15 +68,13 @@ func newPrep(variant operatorv1.ProductVariant) operator.InstallationPrep { Expect(err).NotTo(HaveOccurred()) trustedBundle := certManager.CreateTrustedBundle() - return operator.InstallationPrep{ - Ctx: context.Background(), - Client: c, - Installation: &operatorv1.InstallationSpec{ - Variant: variant, - }, - FelixConfiguration: &v3.FelixConfiguration{}, - CertificateManager: certManager, - TrustedBundle: trustedBundle, - ClusterDomain: "cluster.local", + return []extensions.RenderContextOption{ + extensions.WithContext(context.Background()), + extensions.WithClient(c), + extensions.WithInstallation(&operatorv1.InstallationSpec{Variant: variant}), + extensions.WithFelixConfiguration(&v3.FelixConfiguration{}), + extensions.WithCertificateManager(certManager), + extensions.WithTrustedBundle(trustedBundle), + extensions.WithClusterDomain("cluster.local"), } } diff --git a/pkg/enterprise/node.go b/pkg/enterprise/node.go index f1ea2bcce2..9bf15ece57 100644 --- a/pkg/enterprise/node.go +++ b/pkg/enterprise/node.go @@ -15,7 +15,12 @@ package enterprise import ( + "fmt" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" client "sigs.k8s.io/controller-runtime/pkg/client" @@ -24,47 +29,167 @@ import ( "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/controller/utils" - "github.com/tigera/operator/pkg/operator" + "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" ) const ( + // defaultNodeReporterPort is the port calico/node reports Enterprise internal + // metrics on when FelixConfiguration does not override prometheusReporterPort. defaultNodeReporterPort = 9081 + + // defaultFelixMetricsPort is the Felix prometheus metrics port used when + // FelixConfiguration does not override prometheusMetricsPort. defaultFelixMetricsPort = 9091 + + installCNIContainerName = "install-cni" ) func registerNode() { - operator.OverrideImage(render.ComponentNameNode, func(in *operatorv1.InstallationSpec) (components.Component, bool) { + extensions.OverrideImage(render.ComponentNameNode, func(in *operatorv1.InstallationSpec) (components.Component, bool) { if !in.Variant.IsEnterprise() { return components.Component{}, false } return components.ComponentTigeraNode, true }) - operator.Patch(render.ComponentNameNode, patchNode) + extensions.Modify(render.ComponentNameNode, modifyNode) } -func patchNode(ctx operator.Context, objs []client.Object) []client.Object { +// modifyNode layers Calico Enterprise behavior onto the rendered calico/node +// objects: the extra RBAC rules, the node-metrics Service, and the Enterprise +// daemonset configuration (flow/DNS log env, prometheus reporter, BGP metrics +// readiness check, multi-interface mode, and the calico log volume). +func modifyNode(ctx extensions.RenderContext, objs []client.Object) []client.Object { if ctx.Installation == nil || !ctx.Installation.Variant.IsEnterprise() { return objs } + + if role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, render.CalicoNodeObjectName); ok { + role.Rules = append(role.Rules, nodeEnterpriseRules()...) + } + + // The Network resource is only available in Enterprise / Cloud at this time. + if role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, render.CalicoCNIPluginObjectName); ok { + role.Rules = append(role.Rules, rbacv1.PolicyRule{ + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"networks"}, + Verbs: []string{"get"}, + }) + } + + if ds, ok := extensions.FindObject[*appsv1.DaemonSet](objs, common.NodeDaemonSetName); ok { + modifyNodeDaemonSet(ctx, ds) + } + return append(objs, nodeMetricsService(ctx)) } -// nodeMetricsService builds the enterprise-only calico-node-metrics Service. -// Ports are derived from FelixConfiguration exactly as the installation controller does. -func nodeMetricsService(ctx operator.Context) *corev1.Service { - reporterPort := defaultNodeReporterPort - felixPort := defaultFelixMetricsPort - felixEnabled := false - if fc := ctx.FelixConfiguration; fc != nil { - if fc.Spec.PrometheusReporterPort != nil { - reporterPort = *fc.Spec.PrometheusReporterPort +// nodeEnterpriseRules are the additional cluster role rules calico/node needs in +// Calico Enterprise. +func nodeEnterpriseRules() []rbacv1.PolicyRule { + return []rbacv1.PolicyRule{ + { + // Calico Enterprise needs to be able to read additional resources. + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{ + "bfdconfigurations", + "egressgatewaypolicies", + "externalnetworks", + "licensekeys", + "networks", + "packetcaptures", + "remoteclusterconfigurations", + }, + Verbs: []string{"get", "list", "watch"}, + }, + { + // Tigera Secure updates status for packet captures. + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{ + "packetcaptures", + "packetcaptures/status", + }, + Verbs: []string{"update"}, + }, + } +} + +// modifyNodeDaemonSet applies the Enterprise-specific daemonset changes that the +// base render leaves out: the Enterprise felix env, multi-interface mode, and the +// BGP metrics readiness check. The calico log volume is mounted by the base +// render for both variants, so it is not handled here. +func modifyNodeDaemonSet(ctx extensions.RenderContext, ds *appsv1.DaemonSet) { + spec := &ds.Spec.Template.Spec + + multiInterfaceMode := multiInterfaceModeEnv(ctx.Installation) + + for i := range spec.InitContainers { + if spec.InitContainers[i].Name == installCNIContainerName && multiInterfaceMode != nil { + spec.InitContainers[i].Env = append(spec.InitContainers[i].Env, *multiInterfaceMode) + } + } + + for i := range spec.Containers { + c := &spec.Containers[i] + if c.Name != render.CalicoNodeObjectName { + continue } - if fc.Spec.PrometheusMetricsPort != nil { - felixPort = *fc.Spec.PrometheusMetricsPort + + c.Env = append(c.Env, nodeEnterpriseEnv(ctx)...) + + // Add the BGP metrics readiness check, but only when the base render kept + // the bird readiness check (i.e. BGP is in use and we're not on VPP). + if c.ReadinessProbe != nil && c.ReadinessProbe.Exec != nil && containsString(c.ReadinessProbe.Exec.Command, "--bird-ready") { + c.ReadinessProbe.Exec.Command = append(c.ReadinessProbe.Exec.Command, "--bgp-metrics-ready") } - felixEnabled = utils.IsFelixPrometheusMetricsEnabled(fc) } +} + +// nodeEnterpriseEnv is the Enterprise felix configuration added to the +// calico/node container. +func nodeEnterpriseEnv(ctx extensions.RenderContext) []corev1.EnvVar { + env := []corev1.EnvVar{ + {Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, + {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: fmt.Sprintf("%d", nodeReporterPort(ctx.FelixConfiguration))}, + {Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, + {Name: "FELIX_FLOWLOGSFILEINCLUDELABELS", Value: "true"}, + {Name: "FELIX_FLOWLOGSFILEINCLUDEPOLICIES", Value: "true"}, + {Name: "FELIX_FLOWLOGSFILEINCLUDESERVICE", Value: "true"}, + {Name: "FELIX_FLOWLOGSENABLENETWORKSETS", Value: "true"}, + {Name: "FELIX_FLOWLOGSCOLLECTPROCESSINFO", Value: "true"}, + {Name: "FELIX_DNSLOGSFILEENABLED", Value: "true"}, + {Name: "FELIX_DNSLOGSFILEPERNODELIMIT", Value: "1000"}, + } + + if mode := multiInterfaceModeEnv(ctx.Installation); mode != nil { + env = append(env, *mode) + } + + if ctx.NodePrometheusTLS != nil && ctx.TrustedBundle != nil { + env = append(env, + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERCERTFILE", Value: ctx.NodePrometheusTLS.VolumeMountCertificateFilePath()}, + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERKEYFILE", Value: ctx.NodePrometheusTLS.VolumeMountKeyFilePath()}, + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERCAFILE", Value: ctx.TrustedBundle.MountPath()}, + ) + } + + return env +} + +// multiInterfaceModeEnv returns the MULTI_INTERFACE_MODE env var when the +// installation configures it, or nil otherwise. +func multiInterfaceModeEnv(install *operatorv1.InstallationSpec) *corev1.EnvVar { + if install.CalicoNetwork != nil && install.CalicoNetwork.MultiInterfaceMode != nil { + return &corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: install.CalicoNetwork.MultiInterfaceMode.Value()} + } + return nil +} + +// nodeMetricsService builds the enterprise-only calico-node-metrics Service. +func nodeMetricsService(ctx extensions.RenderContext) *corev1.Service { + reporterPort := nodeReporterPort(ctx.FelixConfiguration) + felixPort := felixMetricsPort(ctx.FelixConfiguration) + felixEnabled := ctx.FelixConfiguration != nil && utils.IsFelixPrometheusMetricsEnabled(ctx.FelixConfiguration) ports := []corev1.ServicePort{ { @@ -103,3 +228,31 @@ func nodeMetricsService(ctx operator.Context) *corev1.Service { }, } } + +// nodeReporterPort returns the reporter metrics port from the FelixConfiguration, +// falling back to the default. The node-metrics Service and the +// FELIX_PROMETHEUSREPORTERPORT env var both derive from here so they can't drift. +func nodeReporterPort(fc *v3.FelixConfiguration) int { + if fc != nil && fc.Spec.PrometheusReporterPort != nil { + return *fc.Spec.PrometheusReporterPort + } + return defaultNodeReporterPort +} + +// felixMetricsPort returns the Felix prometheus metrics port from the +// FelixConfiguration, falling back to the default. +func felixMetricsPort(fc *v3.FelixConfiguration) int { + if fc != nil && fc.Spec.PrometheusMetricsPort != nil { + return *fc.Spec.PrometheusMetricsPort + } + return defaultFelixMetricsPort +} + +func containsString(s []string, v string) bool { + for _, x := range s { + if x == v { + return true + } + } + return false +} diff --git a/pkg/enterprise/node_test.go b/pkg/enterprise/node_test.go index d2370fda6e..2ec7e53b38 100644 --- a/pkg/enterprise/node_test.go +++ b/pkg/enterprise/node_test.go @@ -18,78 +18,178 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" client "sigs.k8s.io/controller-runtime/pkg/client" v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/enterprise" - "github.com/tigera/operator/pkg/operator" + "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" ) var _ = Describe("node enterprise image override", func() { BeforeEach(func() { enterprise.Register() }) - AfterEach(func() { - operator.ResetForTest() - operator.ResetExtensionsForTest() - }) + AfterEach(func() { extensions.ResetForTest() }) It("selects the enterprise node image for the enterprise variant", func() { ent := &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise} - Expect(operator.ResolveImage("node", components.ComponentCalicoNode, ent)).To(Equal(components.ComponentTigeraNode)) + Expect(extensions.ResolveImage("node", components.ComponentCalicoNode, ent)).To(Equal(components.ComponentTigeraNode)) }) It("leaves the default in place for the Calico variant", func() { - oss := &operatorv1.InstallationSpec{Variant: operatorv1.Calico} - Expect(operator.ResolveImage("node", components.ComponentCalicoNode, oss)).To(Equal(components.ComponentCalicoNode)) + calico := &operatorv1.InstallationSpec{Variant: operatorv1.Calico} + Expect(extensions.ResolveImage("node", components.ComponentCalicoNode, calico)).To(Equal(components.ComponentCalicoNode)) }) }) -var _ = Describe("node metrics service modifier", func() { +var _ = Describe("node enterprise modifier", func() { BeforeEach(func() { enterprise.Register() }) - AfterEach(func() { - operator.ResetForTest() - operator.ResetExtensionsForTest() + AfterEach(func() { extensions.ResetForTest() }) + + // newObjs returns the subset of rendered node objects the modifier touches. + newObjs := func() []client.Object { + return []client.Object{ + &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: render.CalicoNodeObjectName}}, + &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: render.CalicoCNIPluginObjectName}}, + &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{Name: common.NodeDaemonSetName}, + Spec: appsv1.DaemonSetSpec{Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + InitContainers: []corev1.Container{{Name: "install-cni"}}, + Containers: []corev1.Container{{ + Name: render.CalicoNodeObjectName, + ReadinessProbe: &corev1.Probe{ProbeHandler: corev1.ProbeHandler{Exec: &corev1.ExecAction{ + Command: []string{"/bin/calico-node", "-bird-ready", "--bird-ready", "--felix-ready"}, + }}}, + }}, + }}}, + }, + } + } + + nodeContainer := func(ds *appsv1.DaemonSet) *corev1.Container { + for i := range ds.Spec.Template.Spec.Containers { + if ds.Spec.Template.Spec.Containers[i].Name == render.CalicoNodeObjectName { + return &ds.Spec.Template.Spec.Containers[i] + } + } + return nil + } + + entCtx := func() extensions.RenderContext { + return extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}} + } + + It("adds the enterprise cluster role rules", func() { + out := extensions.ApplyModifiers(render.ComponentNameNode, entCtx(), newObjs()) + + nodeRole, ok := extensions.FindObject[*rbacv1.ClusterRole](out, render.CalicoNodeObjectName) + Expect(ok).To(BeTrue()) + Expect(nodeRole.Rules).To(ContainElement(HaveField("Resources", ContainElement("licensekeys")))) + + cniRole, ok := extensions.FindObject[*rbacv1.ClusterRole](out, render.CalicoCNIPluginObjectName) + Expect(ok).To(BeTrue()) + Expect(cniRole.Rules).To(ContainElement(HaveField("Resources", ConsistOf("networks")))) + }) + + It("adds the enterprise felix env to the node container", func() { + out := extensions.ApplyModifiers(render.ComponentNameNode, entCtx(), newObjs()) + ds, _ := extensions.FindObject[*appsv1.DaemonSet](out, common.NodeDaemonSetName) + c := nodeContainer(ds) + + Expect(c.Env).To(ContainElements( + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERPORT", Value: "9081"}, + corev1.EnvVar{Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, + corev1.EnvVar{Name: "FELIX_DNSLOGSFILEENABLED", Value: "true"}, + )) + }) + + It("derives the reporter port from FelixConfiguration", func() { + reporter := 7081 + ctx := entCtx() + ctx.FelixConfiguration = &v3.FelixConfiguration{Spec: v3.FelixConfigurationSpec{PrometheusReporterPort: &reporter}} + + out := extensions.ApplyModifiers(render.ComponentNameNode, ctx, newObjs()) + ds, _ := extensions.FindObject[*appsv1.DaemonSet](out, common.NodeDaemonSetName) + Expect(nodeContainer(ds).Env).To(ContainElement(corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERPORT", Value: "7081"})) + }) + + It("appends the BGP metrics readiness check when the bird check is present", func() { + out := extensions.ApplyModifiers(render.ComponentNameNode, entCtx(), newObjs()) + ds, _ := extensions.FindObject[*appsv1.DaemonSet](out, common.NodeDaemonSetName) + Expect(nodeContainer(ds).ReadinessProbe.Exec.Command).To(ContainElement("--bgp-metrics-ready")) }) - It("appends the node metrics service for the enterprise variant", func() { - ctx := operator.Context{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}} - out := operator.ApplyPatches(render.ComponentNameNode, ctx, []client.Object{}) - svc, ok := operator.FindObject[*corev1.Service](out, render.CalicoNodeMetricsService) + It("does not add the BGP metrics readiness check when the bird check is absent", func() { + objs := newObjs() + ds := objs[2].(*appsv1.DaemonSet) + ds.Spec.Template.Spec.Containers[0].ReadinessProbe.Exec.Command = []string{"/bin/calico-node", "--felix-ready"} + + out := extensions.ApplyModifiers(render.ComponentNameNode, entCtx(), objs) + got, _ := extensions.FindObject[*appsv1.DaemonSet](out, common.NodeDaemonSetName) + Expect(nodeContainer(got).ReadinessProbe.Exec.Command).NotTo(ContainElement("--bgp-metrics-ready")) + }) + + It("adds MULTI_INTERFACE_MODE to the node and install-cni containers when configured", func() { + mode := operatorv1.MultiInterfaceModeMultus + ctx := entCtx() + ctx.Installation.CalicoNetwork = &operatorv1.CalicoNetworkSpec{MultiInterfaceMode: &mode} + + out := extensions.ApplyModifiers(render.ComponentNameNode, ctx, newObjs()) + ds, _ := extensions.FindObject[*appsv1.DaemonSet](out, common.NodeDaemonSetName) + + want := corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: mode.Value()} + Expect(nodeContainer(ds).Env).To(ContainElement(want)) + Expect(ds.Spec.Template.Spec.InitContainers[0].Env).To(ContainElement(want)) + }) + + It("appends the node metrics service", func() { + out := extensions.ApplyModifiers(render.ComponentNameNode, entCtx(), newObjs()) + svc, ok := extensions.FindObject[*corev1.Service](out, render.CalicoNodeMetricsService) Expect(ok).To(BeTrue()) - // default ports when FelixConfiguration is nil: 9081 + 9900, felix-metrics-port absent Expect(svc.Spec.Ports).To(HaveLen(2)) Expect(svc.Spec.Ports[0].Port).To(Equal(int32(9081))) Expect(svc.Spec.Ports[1].Port).To(Equal(int32(9900))) }) - It("derives ports and felix-metrics-port from FelixConfiguration", func() { + It("derives metrics service ports and felix-metrics-port from FelixConfiguration", func() { reporter := 7081 metrics := 7091 enabled := true - ctx := operator.Context{ - Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}, - FelixConfiguration: &v3.FelixConfiguration{Spec: v3.FelixConfigurationSpec{ - PrometheusReporterPort: &reporter, - PrometheusMetricsPort: &metrics, - PrometheusMetricsEnabled: &enabled, - }}, - } - out := operator.ApplyPatches(render.ComponentNameNode, ctx, []client.Object{}) - svc, ok := operator.FindObject[*corev1.Service](out, render.CalicoNodeMetricsService) - Expect(ok).To(BeTrue()) + ctx := entCtx() + ctx.FelixConfiguration = &v3.FelixConfiguration{Spec: v3.FelixConfigurationSpec{ + PrometheusReporterPort: &reporter, + PrometheusMetricsPort: &metrics, + PrometheusMetricsEnabled: &enabled, + }} + + out := extensions.ApplyModifiers(render.ComponentNameNode, ctx, newObjs()) + svc, _ := extensions.FindObject[*corev1.Service](out, render.CalicoNodeMetricsService) Expect(svc.Spec.Ports).To(HaveLen(3)) Expect(svc.Spec.Ports[0].Port).To(Equal(int32(7081))) Expect(svc.Spec.Ports[2].Name).To(Equal("felix-metrics-port")) Expect(svc.Spec.Ports[2].Port).To(Equal(int32(7091))) }) - It("does not append it for the Calico variant", func() { - ctx := operator.Context{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}} - out := operator.ApplyPatches(render.ComponentNameNode, ctx, []client.Object{}) - _, ok := operator.FindObject[*corev1.Service](out, render.CalicoNodeMetricsService) + It("is a no-op for the Calico variant", func() { + ctx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}} + out := extensions.ApplyModifiers(render.ComponentNameNode, ctx, newObjs()) + + _, ok := extensions.FindObject[*corev1.Service](out, render.CalicoNodeMetricsService) + Expect(ok).To(BeFalse()) + nodeRole, _ := extensions.FindObject[*rbacv1.ClusterRole](out, render.CalicoNodeObjectName) + Expect(nodeRole.Rules).To(BeEmpty()) + }) + + It("does not panic on a zero RenderContext", func() { + out := extensions.ApplyModifiers(render.ComponentNameNode, extensions.RenderContext{}, newObjs()) + _, ok := extensions.FindObject[*corev1.Service](out, render.CalicoNodeMetricsService) Expect(ok).To(BeFalse()) }) }) diff --git a/pkg/enterprise/typha.go b/pkg/enterprise/typha.go index 8deef34b28..5a2471a5fd 100644 --- a/pkg/enterprise/typha.go +++ b/pkg/enterprise/typha.go @@ -20,20 +20,20 @@ import ( rbacv1 "k8s.io/api/rbac/v1" "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/tigera/operator/pkg/operator" + "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" ) func registerTypha() { - operator.Patch(render.ComponentNameTypha, patchTypha) + extensions.Modify(render.ComponentNameTypha, modifyTypha) } -func patchTypha(ctx operator.Context, objs []client.Object) []client.Object { +func modifyTypha(ctx extensions.RenderContext, objs []client.Object) []client.Object { if ctx.Installation == nil || !ctx.Installation.Variant.IsEnterprise() { return objs } - if role, ok := operator.FindObject[*rbacv1.ClusterRole](objs, "calico-typha"); ok { + if role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, "calico-typha"); ok { role.Rules = append(role.Rules, rbacv1.PolicyRule{ APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, Resources: []string{ @@ -50,7 +50,7 @@ func patchTypha(ctx operator.Context, objs []client.Object) []client.Object { }) } - if dep, ok := operator.FindObject[*appsv1.Deployment](objs, "calico-typha"); ok { + if dep, ok := extensions.FindObject[*appsv1.Deployment](objs, "calico-typha"); ok { net := ctx.Installation.CalicoNetwork if net != nil && net.MultiInterfaceMode != nil { for i := range dep.Spec.Template.Spec.Containers { diff --git a/pkg/enterprise/typha_test.go b/pkg/enterprise/typha_test.go index 9b41d448fa..13fcccc2f9 100644 --- a/pkg/enterprise/typha_test.go +++ b/pkg/enterprise/typha_test.go @@ -25,15 +25,14 @@ import ( operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/enterprise" - "github.com/tigera/operator/pkg/operator" + "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" ) var _ = Describe("typha enterprise modifier", func() { BeforeEach(func() { enterprise.Register() }) AfterEach(func() { - operator.ResetForTest() - operator.ResetExtensionsForTest() + extensions.ResetForTest() }) multiMode := operatorv1.MultiInterfaceModeMultus @@ -51,11 +50,11 @@ var _ = Describe("typha enterprise modifier", func() { } It("adds enterprise RBAC and MULTI_INTERFACE_MODE for the enterprise variant", func() { - ctx := operator.Context{Installation: &operatorv1.InstallationSpec{ + ctx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{ Variant: operatorv1.CalicoEnterprise, CalicoNetwork: &operatorv1.CalicoNetworkSpec{MultiInterfaceMode: &multiMode}, }} - out := operator.ApplyPatches(render.ComponentNameTypha, ctx, newObjs()) + out := extensions.ApplyModifiers(render.ComponentNameTypha, ctx, newObjs()) role := out[0].(*rbacv1.ClusterRole) Expect(role.Rules).To(ContainElement(HaveField("Resources", ContainElement("licensekeys")))) @@ -71,18 +70,18 @@ var _ = Describe("typha enterprise modifier", func() { }) It("is a no-op for the Calico variant", func() { - ctx := operator.Context{Installation: &operatorv1.InstallationSpec{ + ctx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{ Variant: operatorv1.Calico, CalicoNetwork: &operatorv1.CalicoNetworkSpec{MultiInterfaceMode: &multiMode}, }} - out := operator.ApplyPatches(render.ComponentNameTypha, ctx, newObjs()) + out := extensions.ApplyModifiers(render.ComponentNameTypha, ctx, newObjs()) Expect(out[0].(*rbacv1.ClusterRole).Rules).To(BeEmpty()) dep := out[1].(*appsv1.Deployment) Expect(dep.Spec.Template.Spec.Containers[0].Env).To(BeEmpty()) }) It("does not panic on a zero Context (nil Installation)", func() { - out := operator.ApplyPatches(render.ComponentNameTypha, operator.Context{}, newObjs()) + out := extensions.ApplyModifiers(render.ComponentNameTypha, extensions.RenderContext{}, newObjs()) Expect(out[0].(*rbacv1.ClusterRole).Rules).To(BeEmpty()) }) }) diff --git a/pkg/operator/operator_suite_test.go b/pkg/extensions/extensions_suite_test.go similarity index 88% rename from pkg/operator/operator_suite_test.go rename to pkg/extensions/extensions_suite_test.go index 42d3e32f63..791eedb737 100644 --- a/pkg/operator/operator_suite_test.go +++ b/pkg/extensions/extensions_suite_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package operator_test +package extensions_test import ( "testing" @@ -21,7 +21,7 @@ import ( . "github.com/onsi/gomega" ) -func TestOperator(t *testing.T) { +func TestExtensions(t *testing.T) { RegisterFailHandler(Fail) - RunSpecs(t, "pkg/operator Suite") + RunSpecs(t, "pkg/extensions Suite") } diff --git a/pkg/extensions/factory.go b/pkg/extensions/factory.go new file mode 100644 index 0000000000..f6081635ca --- /dev/null +++ b/pkg/extensions/factory.go @@ -0,0 +1,127 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extensions + +import ( + "context" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + "sigs.k8s.io/controller-runtime/pkg/client" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/tls/certificatemanagement" +) + +// Inputs is the reconcile state a RenderContextFactory builds a RenderContext +// from. The installation controller populates it through the With* options. It +// carries both the values that flow straight into the RenderContext and the +// side-effecting dependencies (Client, CertificateManager) a factory needs to +// produce controller-side artifacts. +type Inputs struct { + Ctx context.Context + Client client.Client + Installation *operatorv1.InstallationSpec + FelixConfiguration *v3.FelixConfiguration + CertificateManager certificatemanager.CertificateManager + TrustedBundle certificatemanagement.TrustedBundle + ClusterDomain string +} + +// RenderContextOption sets a field on the Inputs a factory builds from. +type RenderContextOption func(*Inputs) + +func WithContext(ctx context.Context) RenderContextOption { + return func(in *Inputs) { in.Ctx = ctx } +} + +func WithClient(c client.Client) RenderContextOption { + return func(in *Inputs) { in.Client = c } +} + +func WithInstallation(i *operatorv1.InstallationSpec) RenderContextOption { + return func(in *Inputs) { in.Installation = i } +} + +func WithFelixConfiguration(fc *v3.FelixConfiguration) RenderContextOption { + return func(in *Inputs) { in.FelixConfiguration = fc } +} + +func WithCertificateManager(cm certificatemanager.CertificateManager) RenderContextOption { + return func(in *Inputs) { in.CertificateManager = cm } +} + +func WithTrustedBundle(tb certificatemanagement.TrustedBundle) RenderContextOption { + return func(in *Inputs) { in.TrustedBundle = tb } +} + +func WithClusterDomain(d string) RenderContextOption { + return func(in *Inputs) { in.ClusterDomain = d } +} + +// RenderContextFactory builds the RenderContext handed to render modifiers. New +// applies the options, performs any controller-side work (creating +// certificates, extending the trusted bundle), and returns the assembled +// RenderContext - or an error that aborts the reconcile. +// +// This is the generic seam controllers use to extend base operator behavior; +// its first consumer is Calico Enterprise, but nothing here is enterprise +// specific. The core operator default does no side-effecting work. +type RenderContextFactory interface { + New(opts ...RenderContextOption) (RenderContext, error) +} + +// ApplyInputs returns the Inputs produced by applying opts. Factories use it to +// collect the option-supplied state before doing their work. +func ApplyInputs(opts ...RenderContextOption) Inputs { + var in Inputs + for _, o := range opts { + o(&in) + } + return in +} + +// BaseRenderContext maps the generically-gathered inputs onto a RenderContext. +// The default factory and every registered factory build on it, so the base +// fields are assembled in exactly one place. A factory layers its side-effect +// artifacts (e.g. NodePrometheusTLS) on top of the returned value. +func BaseRenderContext(in Inputs) RenderContext { + return RenderContext{ + Installation: in.Installation, + FelixConfiguration: in.FelixConfiguration, + ClusterDomain: in.ClusterDomain, + TrustedBundle: in.TrustedBundle, + } +} + +// defaultFactory is the core operator's RenderContextFactory. It does no +// side-effecting work and returns just the base RenderContext. +type defaultFactory struct{} + +func (defaultFactory) New(opts ...RenderContextOption) (RenderContext, error) { + return BaseRenderContext(ApplyInputs(opts...)), nil +} + +var renderContextFactory RenderContextFactory = defaultFactory{} + +// RegisterRenderContextFactory installs f as the factory the installation +// controller uses. Registration replaces any prior factory, so it is safe to +// call more than once. Without a registered factory the core operator default +// applies. +func RegisterRenderContextFactory(f RenderContextFactory) { renderContextFactory = f } + +// GetRenderContextFactory returns the registered factory, or the core operator +// default when none is registered. +func GetRenderContextFactory() RenderContextFactory { return renderContextFactory } diff --git a/pkg/extensions/factory_test.go b/pkg/extensions/factory_test.go new file mode 100644 index 0000000000..408704b7ad --- /dev/null +++ b/pkg/extensions/factory_test.go @@ -0,0 +1,73 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extensions_test + +import ( + "errors" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/extensions" +) + +var _ = Describe("render context factory", func() { + AfterEach(func() { extensions.ResetForTest() }) + + It("returns the base render context from the default factory", func() { + install := &operatorv1.InstallationSpec{Variant: operatorv1.Calico} + rc, err := extensions.GetRenderContextFactory().New( + extensions.WithInstallation(install), + extensions.WithClusterDomain("cluster.local"), + ) + Expect(err).NotTo(HaveOccurred()) + Expect(rc.Installation).To(BeIdenticalTo(install)) + Expect(rc.ClusterDomain).To(Equal("cluster.local")) + Expect(rc.NodePrometheusTLS).To(BeNil()) + }) + + It("uses a registered factory in place of the default", func() { + extensions.RegisterRenderContextFactory(&fakeFactory{}) + rc, err := extensions.GetRenderContextFactory().New() + Expect(err).NotTo(HaveOccurred()) + Expect(rc.ClusterDomain).To(Equal("from-fake")) + }) + + It("surfaces the factory error", func() { + extensions.RegisterRenderContextFactory(&fakeFactory{err: errors.New("boom")}) + _, err := extensions.GetRenderContextFactory().New() + Expect(err).To(MatchError("boom")) + }) + + It("restores the default factory on reset", func() { + extensions.RegisterRenderContextFactory(&fakeFactory{}) + extensions.ResetForTest() + rc, err := extensions.GetRenderContextFactory().New(extensions.WithClusterDomain("real")) + Expect(err).NotTo(HaveOccurred()) + Expect(rc.ClusterDomain).To(Equal("real")) + }) +}) + +type fakeFactory struct { + err error +} + +func (f *fakeFactory) New(_ ...extensions.RenderContextOption) (extensions.RenderContext, error) { + if f.err != nil { + return extensions.RenderContext{}, f.err + } + return extensions.RenderContext{ClusterDomain: "from-fake"}, nil +} diff --git a/pkg/operator/image.go b/pkg/extensions/image.go similarity index 98% rename from pkg/operator/image.go rename to pkg/extensions/image.go index c6de3b917a..776a02bffd 100644 --- a/pkg/operator/image.go +++ b/pkg/extensions/image.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package operator +package extensions import ( operatorv1 "github.com/tigera/operator/api/v1" diff --git a/pkg/operator/image_test.go b/pkg/extensions/image_test.go similarity index 69% rename from pkg/operator/image_test.go rename to pkg/extensions/image_test.go index b972665e17..5a382d6c36 100644 --- a/pkg/operator/image_test.go +++ b/pkg/extensions/image_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package operator_test +package extensions_test import ( . "github.com/onsi/ginkgo/v2" @@ -20,16 +20,16 @@ import ( operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/components" - "github.com/tigera/operator/pkg/operator" + "github.com/tigera/operator/pkg/extensions" ) var _ = Describe("image overrides", func() { AfterEach(func() { - operator.ResetForTest() + extensions.ResetForTest() }) It("uses the override when one matches", func() { - operator.OverrideImage("node", func(in *operatorv1.InstallationSpec) (components.Component, bool) { + extensions.OverrideImage("node", func(in *operatorv1.InstallationSpec) (components.Component, bool) { if !in.Variant.IsEnterprise() { return components.Component{}, false } @@ -37,11 +37,11 @@ var _ = Describe("image overrides", func() { }) ent := &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise} - Expect(operator.ResolveImage("node", components.ComponentCalicoNode, ent)).To(Equal(components.ComponentTigeraNode)) + Expect(extensions.ResolveImage("node", components.ComponentCalicoNode, ent)).To(Equal(components.ComponentTigeraNode)) }) It("falls back to the default when no override matches", func() { - oss := &operatorv1.InstallationSpec{Variant: operatorv1.Calico} - Expect(operator.ResolveImage("node", components.ComponentCalicoNode, oss)).To(Equal(components.ComponentCalicoNode)) + calico := &operatorv1.InstallationSpec{Variant: operatorv1.Calico} + Expect(extensions.ResolveImage("node", components.ComponentCalicoNode, calico)).To(Equal(components.ComponentCalicoNode)) }) }) diff --git a/pkg/operator/patch.go b/pkg/extensions/modifier.go similarity index 50% rename from pkg/operator/patch.go rename to pkg/extensions/modifier.go index bce2ff783c..5b911edc72 100644 --- a/pkg/operator/patch.go +++ b/pkg/extensions/modifier.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package operator +package extensions import ( "sigs.k8s.io/controller-runtime/pkg/client" @@ -20,22 +20,26 @@ import ( "github.com/tigera/operator/pkg/imageoverride" ) -// PatchFunc post-processes the objects a render component produced. It may -// mutate matched objects and/or append additional objects, and must return the +// Modifier post-processes the objects a render component produced. It may mutate +// matched objects and/or append additional objects, and must return the // (possibly extended) slice. Implementations self-gate on ctx.Installation.Variant. -type PatchFunc func(ctx Context, objs []client.Object) []client.Object +type Modifier func(ctx RenderContext, objs []client.Object) []client.Object -var patches = map[string][]PatchFunc{} +var modifiers = map[string]Modifier{} -// Patch registers fn to run against the named component's objects. Multiple -// patches may be registered for the same component; they run in registration order. -func Patch(component string, fn PatchFunc) { - patches[component] = append(patches[component], fn) +// Modify registers fn as the modifier for the named component. A component has +// at most one modifier; the modifier handles all of that component's +// extension-specific mutations. Registration replaces any prior modifier, so it +// is idempotent and safe to call more than once - matching the image-override +// registry rather than stacking duplicate work. +func Modify(component string, fn Modifier) { + modifiers[component] = fn } -// ApplyPatches runs every patch registered for the named component over objs. -func ApplyPatches(component string, ctx Context, objs []client.Object) []client.Object { - for _, fn := range patches[component] { +// ApplyModifiers runs the modifier registered for the named component over objs, +// returning objs unchanged when none is registered. +func ApplyModifiers(component string, ctx RenderContext, objs []client.Object) []client.Object { + if fn, ok := modifiers[component]; ok { objs = fn(ctx, objs) } return objs @@ -52,8 +56,10 @@ func FindObject[T client.Object](objs []client.Object, name string) (T, bool) { return zero, false } -// ResetForTest clears all registries. Test-only. +// ResetForTest clears every registry: modifiers, image overrides, and the +// render context factory. Test-only. func ResetForTest() { - patches = map[string][]PatchFunc{} + modifiers = map[string]Modifier{} + renderContextFactory = defaultFactory{} imageoverride.ResetForTest() } diff --git a/pkg/operator/patch_test.go b/pkg/extensions/modifier_test.go similarity index 53% rename from pkg/operator/patch_test.go rename to pkg/extensions/modifier_test.go index d9cd4de108..7b40beb740 100644 --- a/pkg/operator/patch_test.go +++ b/pkg/extensions/modifier_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package operator_test +package extensions_test import ( . "github.com/onsi/ginkgo/v2" @@ -21,24 +21,24 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/tigera/operator/pkg/operator" + "github.com/tigera/operator/pkg/extensions" ) -var _ = Describe("patch registry", func() { +var _ = Describe("modifier registry", func() { AfterEach(func() { - operator.ResetForTest() + extensions.ResetForTest() }) - It("applies a registered patch to the matching component", func() { - operator.Patch("test", func(ctx operator.Context, objs []client.Object) []client.Object { - cm, ok := operator.FindObject[*corev1.ConfigMap](objs, "cm") + It("applies a registered modifier to the matching component", func() { + extensions.Modify("test", func(ctx extensions.RenderContext, objs []client.Object) []client.Object { + cm, ok := extensions.FindObject[*corev1.ConfigMap](objs, "cm") Expect(ok).To(BeTrue()) cm.Data = map[string]string{"k": "v"} return append(objs, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "extra"}}) }) in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} - out := operator.ApplyPatches("test", operator.Context{}, in) + out := extensions.ApplyModifiers("test", extensions.RenderContext{}, in) Expect(out).To(HaveLen(2)) cm := out[0].(*corev1.ConfigMap) @@ -46,9 +46,23 @@ var _ = Describe("patch registry", func() { Expect(out[1].GetName()).To(Equal("extra")) }) - It("returns objects unchanged when no patch is registered", func() { + It("returns objects unchanged when no modifier is registered", func() { in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} - out := operator.ApplyPatches("unregistered", operator.Context{}, in) + out := extensions.ApplyModifiers("unregistered", extensions.RenderContext{}, in) Expect(out).To(Equal(in)) }) + + It("replaces rather than stacks when a component is registered twice", func() { + add := func(name string) extensions.Modifier { + return func(_ extensions.RenderContext, objs []client.Object) []client.Object { + return append(objs, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: name}}) + } + } + extensions.Modify("test", add("first")) + extensions.Modify("test", add("second")) + + out := extensions.ApplyModifiers("test", extensions.RenderContext{}, nil) + Expect(out).To(HaveLen(1)) + Expect(out[0].GetName()).To(Equal("second")) + }) }) diff --git a/pkg/operator/context.go b/pkg/extensions/rendercontext.go similarity index 70% rename from pkg/operator/context.go rename to pkg/extensions/rendercontext.go index cb84bbd052..52d0d753c5 100644 --- a/pkg/operator/context.go +++ b/pkg/extensions/rendercontext.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package operator +package extensions import ( v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" @@ -20,14 +20,15 @@ import ( "github.com/tigera/operator/pkg/tls/certificatemanagement" ) -// Context carries reconcile-derived inputs from controllers into render -// modifiers. OSS code never reads these fields - only registered modifiers do. +// RenderContext carries reconcile-derived inputs from controllers into render +// modifiers. Core operator code never reads these fields - only registered +// modifiers do. // Two kinds of value live here: // - raw cluster state gathered generically (Installation, FelixConfiguration, // ClusterDomain) that modifiers derive their own values from, and // - controller-produced artifacts (TrustedBundle, NodePrometheusTLS) that can // only be created controller-side because they have cluster side effects. -type Context struct { +type RenderContext struct { Installation *operatorv1.InstallationSpec FelixConfiguration *v3.FelixConfiguration ClusterDomain string @@ -35,7 +36,10 @@ type Context struct { // TrustedBundle is the shared CA bundle for the calico-system namespace. TrustedBundle certificatemanagement.TrustedBundle - // NodePrometheusTLS is produced by the installation controller's enterprise - // extension and mounted by the node modifier. + // NodePrometheusTLS is created by the enterprise RenderContextFactory (it has + // cluster side effects, so it can't be built in a modifier). Two consumers + // read it: the installation controller passes it to the node configuration + // (PrometheusServerTLS) so the keypair is mounted, and the node modifier uses + // it to set the FELIX_PROMETHEUSREPORTER* certificate env vars. NodePrometheusTLS certificatemanagement.KeyPairInterface } diff --git a/pkg/operator/extension.go b/pkg/operator/extension.go deleted file mode 100644 index de5080c45d..0000000000 --- a/pkg/operator/extension.go +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright (c) 2026 Tigera, Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package operator - -import ( - "context" - - v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" - operatorv1 "github.com/tigera/operator/api/v1" - "github.com/tigera/operator/pkg/controller/certificatemanager" - "github.com/tigera/operator/pkg/tls/certificatemanagement" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -// InstallationPrep is the input to an InstallationExtension's Prepare. It holds -// the generically-gathered reconcile state the extension needs to do its -// side-effecting work (create certs, assemble the trusted bundle). -type InstallationPrep struct { - Ctx context.Context - Client client.Client - Installation *operatorv1.InstallationSpec - FelixConfiguration *v3.FelixConfiguration - CertificateManager certificatemanager.CertificateManager - TrustedBundle certificatemanagement.TrustedBundle - ClusterDomain string -} - -// InstallationExtension is the enterprise hook for the installation controller. -// Prepare runs controller-side before rendering. It performs work modifiers -// can't (cluster side effects, fetching/creating certificates) and may abort -// the reconcile by returning an error. It returns the Context handed to the -// render patches; on the Calico variant it should return an empty Context and -// nil error. -type InstallationExtension interface { - Prepare(p InstallationPrep) (Context, error) -} - -var installationExtension InstallationExtension - -// RegisterInstallationExtension registers the installation controller extension. -func RegisterInstallationExtension(e InstallationExtension) { installationExtension = e } - -// GetInstallationExtension returns the registered extension, or nil. -func GetInstallationExtension() InstallationExtension { return installationExtension } - -// ResetExtensionsForTest clears registered extensions. Test-only. -func ResetExtensionsForTest() { installationExtension = nil } diff --git a/pkg/operator/extension_test.go b/pkg/operator/extension_test.go deleted file mode 100644 index e3ecd95592..0000000000 --- a/pkg/operator/extension_test.go +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright (c) 2026 Tigera, Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package operator_test - -import ( - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "github.com/tigera/operator/pkg/operator" -) - -var _ = Describe("controller extensions", func() { - AfterEach(func() { operator.ResetExtensionsForTest() }) - - It("returns the registered installation extension", func() { - ext := &fakeInstallationExtension{} - operator.RegisterInstallationExtension(ext) - Expect(operator.GetInstallationExtension()).To(BeIdenticalTo(ext)) - }) - - It("returns nil when none is registered", func() { - Expect(operator.GetInstallationExtension()).To(BeNil()) - }) -}) - -type fakeInstallationExtension struct{} - -func (f *fakeInstallationExtension) Prepare(_ operator.InstallationPrep) (operator.Context, error) { - return operator.Context{}, nil -} diff --git a/pkg/render/component.go b/pkg/render/component.go index 585355438d..0fa4f5da8b 100644 --- a/pkg/render/component.go +++ b/pkg/render/component.go @@ -40,17 +40,20 @@ type Component interface { SupportedOSType() rmeta.OSType } -// Named is implemented by components that expose enterprise extension points. -// The componentHandler uses Name() to look up registered patches. Components -// without enterprise extensions need not implement it. -type Named interface { +// Extensible is implemented by components that expose extension points. The +// componentHandler uses Name() to look up registered modifiers. Components +// without extensions need not implement it. +// +// Note this interface is structural: any component that grows a Name() string +// method for an unrelated reason becomes modifier-eligible. There are no name +// collisions today, but keep it in mind when adding a Name() method. +type Extensible interface { Name() string } -// Component names used as keys into the operator patch registry. Keep these in -// sync with the Name() methods that return them. +// Component names used as keys into the extension modifier registry. Keep these +// in sync with the Name() methods that return them. const ( - ComponentNameTypha = "typha" - ComponentNameNode = "node" - ComponentNameAPIServer = "apiserver" + ComponentNameTypha = "typha" + ComponentNameNode = "node" ) diff --git a/pkg/render/enterprise_setup_test.go b/pkg/render/enterprise_setup_test.go index 5f464c950b..13766f7035 100644 --- a/pkg/render/enterprise_setup_test.go +++ b/pkg/render/enterprise_setup_test.go @@ -20,10 +20,15 @@ import ( "github.com/tigera/operator/pkg/enterprise" ) -// The render suite asserts enterprise-variant output for components whose -// variant-specific behavior now lives in registered modifiers/overrides -// (e.g. the node image). Register them once so the suite exercises the same -// integrated behavior the operator binary produces. +// Register the enterprise extensions once for the whole render suite. This wires +// two things the suite relies on: +// - the image override, which the Objects()-level render tests pick up through +// ResolveImages (e.g. the enterprise node image), and +// - the modifiers, which node_enterprise_test.go applies explicitly to real +// render output to check they still match it. +// +// The plain Objects()-level tests do not run modifiers - those only run at the +// componentHandler - so registering here does not change their output. var _ = BeforeSuite(func() { enterprise.Register() }) diff --git a/pkg/render/node.go b/pkg/render/node.go index 6bf69d2f23..0e42fe3091 100644 --- a/pkg/render/node.go +++ b/pkg/render/node.go @@ -116,11 +116,10 @@ type NodeConfiguration struct { GoldmaneIP string // Optional fields. - LogCollector *operatorv1.LogCollector - MigrateNamespaces bool - NodeAppArmorProfile string - BirdTemplates map[string]string - NodeReporterMetricsPort int + LogCollector *operatorv1.LogCollector + MigrateNamespaces bool + NodeAppArmorProfile string + BirdTemplates map[string]string // CanRemoveCNIFinalizer specifies whether CNI plugin is still needed during uninstall since the CNI plugin and // associated RBAC resources are required for pod teardown to succeed. Setting this to true removes @@ -147,10 +146,6 @@ type NodeConfiguration struct { // should this value change. BindMode string - FelixPrometheusMetricsEnabled bool - - FelixPrometheusMetricsPort int - V3CRDs bool } @@ -554,34 +549,6 @@ func (c *nodeComponent) nodeRole() *rbacv1.ClusterRole { }, }, } - if c.cfg.Installation.Variant.IsEnterprise() { - extraRules := []rbacv1.PolicyRule{ - { - // Calico Enterprise needs to be able to read additional resources. - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{ - "bfdconfigurations", - "egressgatewaypolicies", - "externalnetworks", - "licensekeys", - "networks", - "packetcaptures", - "remoteclusterconfigurations", - }, - Verbs: []string{"get", "list", "watch"}, - }, - { - // Tigera Secure updates status for packet captures. - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{ - "packetcaptures", - "packetcaptures/status", - }, - Verbs: []string{"update"}, - }, - } - role.Rules = append(role.Rules, extraRules...) - } if c.cfg.Installation.KubernetesProvider.IsOpenShift() { role.Rules = append(role.Rules, rbacv1.PolicyRule{ APIGroups: []string{"security.openshift.io"}, @@ -643,14 +610,6 @@ func (c *nodeComponent) cniPluginRole() *rbacv1.ClusterRole { }, }, } - if c.cfg.Installation.Variant.IsEnterprise() { - // The Network resource is only available in Enterprise / Cloud at this time. - role.Rules = append(role.Rules, rbacv1.PolicyRule{ - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"networks"}, - Verbs: []string{"get"}, - }) - } return role } @@ -1094,13 +1053,16 @@ func (c *nodeComponent) nodeVolumes() []corev1.Volume { c.cfg.TLS.TrustedBundle.Volume(), c.cfg.TLS.NodeSecret.Volume(), c.varRunCalicoVolume(), - corev1.Volume{Name: "var-lib-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/lib/calico", Type: &dirOrCreate}}}, + {Name: "var-lib-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/lib/calico", Type: &dirOrCreate}}}, + // The Calico log directory. The CNI plugin logs to the cni/ subdirectory of + // this, and Felix writes its flow/DNS logs here on the enterprise variant. + {Name: "var-log-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}}, // Volume for the containing directory so that the init container can mount the child bpf directory if needed. - corev1.Volume{Name: "sys-fs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs", Type: &dirOrCreate}}}, + {Name: "sys-fs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs", Type: &dirOrCreate}}}, // Volume for the bpffs itself, used by the main node container. - corev1.Volume{Name: "bpffs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs/bpf", Type: &dirMustExist}}}, + {Name: "bpffs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs/bpf", Type: &dirMustExist}}}, // Volume used by mount-cgroupv2 init container to access root cgroup name space of node. - corev1.Volume{Name: "nodeproc", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/proc"}}}, + {Name: "nodeproc", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/proc"}}}, } if c.vppDataplaneEnabled() { @@ -1114,17 +1076,6 @@ func (c *nodeComponent) nodeVolumes() []corev1.Volume { if c.cfg.Installation.CNI.Type == operatorv1.PluginCalico { volumes = append(volumes, corev1.Volume{Name: "cni-bin-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: *c.cfg.Installation.CNI.BinDir, Type: &dirOrCreate}}}) volumes = append(volumes, corev1.Volume{Name: "cni-net-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: *c.cfg.Installation.CNI.ConfDir}}}) - volumes = append(volumes, corev1.Volume{Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico/cni"}}}) - } - - // Override with Tigera-specific config. - if c.cfg.Installation.Variant.IsEnterprise() { - // Add volume for calico logs. - calicoLogVol := corev1.Volume{ - Name: "var-log-calico", - VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}, - } - volumes = append(volumes, calicoLogVol) } // Create and append flexvolume @@ -1326,12 +1277,6 @@ func (c *nodeComponent) cniEnvvars() []corev1.EnvVar { envVars = append(envVars, c.cfg.K8sServiceEp.EnvVars()...) - if c.cfg.Installation.Variant.IsEnterprise() { - if c.cfg.Installation.CalicoNetwork != nil && c.cfg.Installation.CalicoNetwork.MultiInterfaceMode != nil { - envVars = append(envVars, corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: c.cfg.Installation.CalicoNetwork.MultiInterfaceMode.Value()}) - } - } - return envVars } @@ -1375,15 +1320,9 @@ func (c *nodeComponent) nodeVolumeMounts() []corev1.VolumeMount { if c.vppDataplaneEnabled() { nodeVolumeMounts = append(nodeVolumeMounts, corev1.VolumeMount{MountPath: "/usr/local/bin/felix-plugins", Name: "felix-plugins", ReadOnly: true}) } - if c.cfg.Installation.Variant.IsEnterprise() { - extraNodeMounts := []corev1.VolumeMount{ - {MountPath: "/var/log/calico", Name: "var-log-calico"}, - } - nodeVolumeMounts = append(nodeVolumeMounts, extraNodeMounts...) - } else if c.cfg.Installation.CNI.Type == operatorv1.PluginCalico { - cniLogMount := corev1.VolumeMount{MountPath: "/var/log/calico/cni", Name: "cni-log-dir", ReadOnly: false} - nodeVolumeMounts = append(nodeVolumeMounts, cniLogMount) - } + // Mount the Calico log directory. The CNI plugin writes to the cni/ subdirectory + // and, on the enterprise variant, Felix writes its flow/DNS logs here too. + nodeVolumeMounts = append(nodeVolumeMounts, corev1.VolumeMount{MountPath: "/var/log/calico", Name: "var-log-calico"}) if c.cfg.Installation.CNI.Type == operatorv1.PluginCalico { nodeVolumeMounts = append(nodeVolumeMounts, corev1.VolumeMount{MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}) @@ -1629,35 +1568,6 @@ func (c *nodeComponent) nodeEnvVars() []corev1.EnvVar { nodeEnv = append(nodeEnv, corev1.EnvVar{Name: "FELIX_IPV6SUPPORT", Value: "false"}) } - if c.cfg.Installation.Variant.IsEnterprise() { - // Add in Calico Enterprise specific configuration. - extraNodeEnv := []corev1.EnvVar{ - {Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, - {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: fmt.Sprintf("%d", c.cfg.NodeReporterMetricsPort)}, - {Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDELABELS", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDEPOLICIES", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDESERVICE", Value: "true"}, - {Name: "FELIX_FLOWLOGSENABLENETWORKSETS", Value: "true"}, - {Name: "FELIX_FLOWLOGSCOLLECTPROCESSINFO", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEPERNODELIMIT", Value: "1000"}, - } - - if c.cfg.Installation.CalicoNetwork != nil && c.cfg.Installation.CalicoNetwork.MultiInterfaceMode != nil { - extraNodeEnv = append(extraNodeEnv, corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: c.cfg.Installation.CalicoNetwork.MultiInterfaceMode.Value()}) - } - - if c.cfg.PrometheusServerTLS != nil { - extraNodeEnv = append(extraNodeEnv, - corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERCERTFILE", Value: c.cfg.PrometheusServerTLS.VolumeMountCertificateFilePath()}, - corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERKEYFILE", Value: c.cfg.PrometheusServerTLS.VolumeMountKeyFilePath()}, - corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERCAFILE", Value: c.cfg.TLS.TrustedBundle.MountPath()}, - ) - } - nodeEnv = append(nodeEnv, extraNodeEnv...) - } - if c.cfg.Installation.NodeMetricsPort != nil { // If a node metrics port was given, then enable felix prometheus metrics and set the port. // Note that this takes precedence over any FelixConfiguration resources in the cluster. @@ -1736,10 +1646,7 @@ func (c *nodeComponent) nodeLivenessReadinessProbes() (*corev1.Probe, *corev1.Pr var readinessCmd []string readinessCmd = []string{components.CalicoBinaryPath, "component", "node", "health", "--bird-ready", "--felix-ready"} - if c.cfg.Installation.Variant.IsEnterprise() { - readinessCmd = append(readinessCmd, "--bgp-metrics-ready") - } - // If not using BGP or using VPP, don't check bird status (or bgp metrics server for enterprise). + // If not using BGP or using VPP, don't check bird status. if !bgpEnabled(c.cfg.Installation) || c.vppDataplaneEnabled() { readinessCmd = []string{components.CalicoBinaryPath, "component", "node", "health", "--felix-ready"} } diff --git a/pkg/render/node_enterprise_test.go b/pkg/render/node_enterprise_test.go new file mode 100644 index 0000000000..4cc73d2a70 --- /dev/null +++ b/pkg/render/node_enterprise_test.go @@ -0,0 +1,172 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package render_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/apis" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/controller/k8sapi" + ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/render" +) + +// These tests run the real node/typha render output through the registered +// enterprise modifiers. The render suite registers the enterprise extensions in +// its BeforeSuite, so this exercises the same integrated behavior the operator +// binary produces - and, importantly, catches a modifier whose FindObject stops +// matching because render renamed an object or container. +var _ = Describe("node enterprise modifier integration", func() { + var ( + cli client.Client + certManager certificatemanager.CertificateManager + typhaNodeTLS *render.TyphaNodeTLS + instance *operatorv1.InstallationSpec + renderCtx extensions.RenderContext + ) + + nodeContainer := func(ds *appsv1.DaemonSet) *corev1.Container { + for i := range ds.Spec.Template.Spec.Containers { + if ds.Spec.Template.Spec.Containers[i].Name == render.CalicoNodeObjectName { + return &ds.Spec.Template.Spec.Containers[i] + } + } + return nil + } + + BeforeEach(func() { + scheme := runtime.NewScheme() + Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) + cli = ctrlrfake.DefaultFakeClientBuilder(scheme).Build() + + var err error + certManager, err = certificatemanager.Create(cli, nil, "", common.OperatorNamespace(), certificatemanager.AllowCACreation()) + Expect(err).NotTo(HaveOccurred()) + typhaNodeTLS = getTyphaNodeTLS(cli, certManager) + + nodePrometheusTLS, err := certManager.GetOrCreateKeyPair(cli, render.NodePrometheusTLSServerSecret, common.OperatorNamespace(), []string{"calico-node-metrics"}) + Expect(err).NotTo(HaveOccurred()) + typhaNodeTLS.TrustedBundle.AddCertificates(nodePrometheusTLS) + + confDir, binDir := render.DefaultCNIDirectories(operatorv1.ProviderNone) + bgp := operatorv1.BGPEnabled + instance = &operatorv1.InstallationSpec{ + Variant: operatorv1.CalicoEnterprise, + CNI: &operatorv1.CNISpec{ + Type: operatorv1.PluginCalico, + IPAM: &operatorv1.IPAMSpec{Type: operatorv1.IPAMPluginCalico}, + BinDir: &binDir, + ConfDir: &confDir, + }, + CalicoNetwork: &operatorv1.CalicoNetworkSpec{ + BGP: &bgp, + IPPools: []operatorv1.IPPool{{CIDR: "192.168.1.0/16"}}, + }, + } + + renderCtx = extensions.RenderContext{ + Installation: instance, + TrustedBundle: typhaNodeTLS.TrustedBundle, + NodePrometheusTLS: nodePrometheusTLS, + } + }) + + // renderNodeObjects renders the real node component and applies the registered + // modifier, exactly as the componentHandler does. + renderNodeObjects := func() []client.Object { + cfg := &render.NodeConfiguration{ + K8sServiceEp: k8sapi.ServiceEndpoint{}, + Installation: instance, + TLS: typhaNodeTLS, + ClusterDomain: dns.DefaultClusterDomain, + FelixHealthPort: 9099, + IPPools: instance.CalicoNetwork.IPPools, + PrometheusServerTLS: renderCtx.NodePrometheusTLS, + } + comp := render.Node(cfg) + Expect(comp.ResolveImages(nil)).NotTo(HaveOccurred()) + objs, _ := comp.Objects() + return extensions.ApplyModifiers(render.ComponentNameNode, renderCtx, objs) + } + + It("appends the node metrics service to the real render output", func() { + objs := renderNodeObjects() + svc, ok := extensions.FindObject[*corev1.Service](objs, render.CalicoNodeMetricsService) + Expect(ok).To(BeTrue(), "expected the modifier to append %s", render.CalicoNodeMetricsService) + Expect(svc.Namespace).To(Equal(common.CalicoNamespace)) + }) + + It("adds the enterprise rules to the real cluster roles", func() { + objs := renderNodeObjects() + + nodeRole, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, render.CalicoNodeObjectName) + Expect(ok).To(BeTrue()) + Expect(nodeRole.Rules).To(ContainElement(HaveField("Resources", ContainElement("licensekeys")))) + + cniRole, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, render.CalicoCNIPluginObjectName) + Expect(ok).To(BeTrue()) + Expect(cniRole.Rules).To(ContainElement(HaveField("Resources", ContainElement("networks")))) + }) + + It("rewrites the real node daemonset for enterprise", func() { + objs := renderNodeObjects() + ds, ok := extensions.FindObject[*appsv1.DaemonSet](objs, common.NodeDaemonSetName) + Expect(ok).To(BeTrue()) + + c := nodeContainer(ds) + Expect(c).NotTo(BeNil()) + + Expect(c.Env).To(ContainElements( + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, + corev1.EnvVar{Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, + )) + // The reporter cert env is wired from the NodePrometheusTLS keypair the + // factory creates. + Expect(c.Env).To(ContainElement(HaveField("Name", "FELIX_PROMETHEUSREPORTERCERTFILE"))) + + // BGP is enabled, so the bird readiness check is present and the modifier + // adds the BGP metrics check. + Expect(c.ReadinessProbe.Exec.Command).To(ContainElement("--bgp-metrics-ready")) + }) + + It("adds the enterprise rules to the real typha cluster role", func() { + comp := render.Typha(&render.TyphaConfiguration{ + K8sServiceEp: k8sapi.ServiceEndpoint{}, + Installation: instance, + TLS: typhaNodeTLS, + ClusterDomain: dns.DefaultClusterDomain, + FelixHealthPort: 9099, + }) + Expect(comp.ResolveImages(nil)).NotTo(HaveOccurred()) + objs, _ := comp.Objects() + objs = extensions.ApplyModifiers(render.ComponentNameTypha, renderCtx, objs) + + role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, "calico-typha") + Expect(ok).To(BeTrue()) + Expect(role.Rules).To(ContainElement(HaveField("Resources", ContainElement("licensekeys")))) + }) +}) diff --git a/pkg/render/node_test.go b/pkg/render/node_test.go index 697feda36d..c534759543 100644 --- a/pkg/render/node_test.go +++ b/pkg/render/node_test.go @@ -134,14 +134,12 @@ var _ = Describe("Node rendering tests", func() { // Create a default configuration. cfg = render.NodeConfiguration{ - K8sServiceEp: k8sServiceEp, - Installation: defaultInstance, - TLS: typhaNodeTLS, - ClusterDomain: defaultClusterDomain, - FelixHealthPort: 9099, - IPPools: defaultInstance.CalicoNetwork.IPPools, - FelixPrometheusMetricsEnabled: false, - FelixPrometheusMetricsPort: 9098, + K8sServiceEp: k8sServiceEp, + Installation: defaultInstance, + TLS: typhaNodeTLS, + ClusterDomain: defaultClusterDomain, + FelixHealthPort: 9099, + IPPools: defaultInstance.CalicoNetwork.IPPools, } }) @@ -321,7 +319,7 @@ var _ = Describe("Node rendering tests", func() { {Name: "xtables-lock", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/run/xtables.lock", Type: &fileOrCreate}}}, {Name: "cni-bin-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/opt/cni/bin", Type: &dirOrCreate}}}, {Name: "cni-net-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/etc/cni/net.d"}}}, - {Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico/cni"}}}, + {Name: "var-log-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}}, {Name: "policysync", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/nodeagent", Type: &dirOrCreate}}}, {Name: "sys-fs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs", Type: &dirOrCreate}}}, {Name: "bpffs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs/bpf", Type: &dirMustExist}}}, @@ -359,7 +357,7 @@ var _ = Describe("Node rendering tests", func() { {MountPath: "/var/run/nodeagent", Name: "policysync"}, {MountPath: "/etc/pki/tls/certs", Name: "tigera-ca-bundle", ReadOnly: true}, {MountPath: "/node-certs", Name: render.NodeTLSSecretName, ReadOnly: true}, - {MountPath: "/var/log/calico/cni", Name: "cni-log-dir", ReadOnly: false}, + {MountPath: "/var/log/calico", Name: "var-log-calico"}, {MountPath: "/sys/fs/bpf", Name: "bpffs"}, } Expect(ds.Spec.Template.Spec.Containers[0].VolumeMounts).To(ConsistOf(expectedNodeVolumeMounts)) @@ -367,7 +365,7 @@ var _ = Describe("Node rendering tests", func() { // Verify tolerations. Expect(ds.Spec.Template.Spec.Tolerations).To(ConsistOf(rmeta.TolerateAll)) - verifyProbesAndLifecycle(ds, false, false) + verifyProbesAndLifecycle(ds, false) }) It("should render node correctly for BPF dataplane", func() { @@ -512,7 +510,7 @@ var _ = Describe("Node rendering tests", func() { {Name: "xtables-lock", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/run/xtables.lock", Type: &fileOrCreate}}}, {Name: "cni-bin-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/opt/cni/bin", Type: &dirOrCreate}}}, {Name: "cni-net-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/etc/cni/net.d"}}}, - {Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico/cni"}}}, + {Name: "var-log-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}}, {Name: "sys-fs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs", Type: &dirOrCreate}}}, {Name: "bpffs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs/bpf", Type: &dirMustExist}}}, {Name: "nodeproc", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/proc"}}}, @@ -550,7 +548,7 @@ var _ = Describe("Node rendering tests", func() { {MountPath: "/var/run/nodeagent", Name: "policysync"}, {MountPath: "/etc/pki/tls/certs", Name: "tigera-ca-bundle", ReadOnly: true}, {MountPath: "/node-certs", Name: render.NodeTLSSecretName, ReadOnly: true}, - {MountPath: "/var/log/calico/cni", Name: "cni-log-dir", ReadOnly: false}, + {MountPath: "/var/log/calico", Name: "var-log-calico"}, {MountPath: "/sys/fs/bpf", Name: "bpffs"}, } Expect(ds.Spec.Template.Spec.Containers[0].VolumeMounts).To(ConsistOf(expectedNodeVolumeMounts)) @@ -558,7 +556,7 @@ var _ = Describe("Node rendering tests", func() { // Verify tolerations. Expect(ds.Spec.Template.Spec.Tolerations).To(ConsistOf(rmeta.TolerateAll)) - verifyProbesAndLifecycle(ds, false, false) + verifyProbesAndLifecycle(ds, false) }) It("should properly render an explicitly configured MTU", func() { @@ -637,103 +635,6 @@ var _ = Describe("Node rendering tests", func() { } }) - It("should render all resources for a default configuration using CalicoEnterprise", func() { - expectedResources := []struct { - name string - ns string - group string - version string - kind string - }{ - {name: "calico-node", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ServiceAccount"}, - {name: "calico-node", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, - {name: "calico-node", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: "calico-cni-plugin", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ServiceAccount"}, - {name: "calico-cni-plugin", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, - {name: "calico-cni-plugin", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: "cni-config", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ConfigMap"}, - {name: common.NodeDaemonSetName, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "DaemonSet"}, - } - defaultInstance.Variant = operatorv1.CalicoEnterprise - cfg.NodeReporterMetricsPort = 9081 - - component := render.Node(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - Expect(len(resources)).To(Equal(len(expectedResources))) - - // Should render the correct resources. - i := 0 - for _, expectedRes := range expectedResources { - rtest.ExpectResourceTypeAndObjectMetadata(resources[i], expectedRes.name, expectedRes.ns, expectedRes.group, expectedRes.version, expectedRes.kind) - i++ - } - - // The DaemonSet should have the correct configuration. - ds := rtest.GetResource(resources, "calico-node", "calico-system", "apps", "v1", "DaemonSet").(*appsv1.DaemonSet) - - // The pod template should have node critical priority - Expect(ds.Spec.Template.Spec.PriorityClassName).To(Equal(render.NodePriorityClassName)) - Expect(ds.Spec.Template.Spec.Containers[0].Image).To(Equal(components.TigeraRegistry + "tigera/node:" + components.ComponentTigeraNode.Version)) - verifyInitContainers(ds, defaultInstance) - - expectedNodeEnv := []corev1.EnvVar{ - // Default envvars. - {Name: "DATASTORE_TYPE", Value: "kubernetes"}, - {Name: "WAIT_FOR_DATASTORE", Value: "true"}, - {Name: "CALICO_MANAGE_CNI", Value: "true"}, - {Name: "CALICO_NETWORKING_BACKEND", Value: "bird"}, - {Name: "CLUSTER_TYPE", Value: "k8s,operator,bgp"}, - {Name: "CALICO_DISABLE_FILE_LOGGING", Value: "false"}, - {Name: "FELIX_DEFAULTENDPOINTTOHOSTACTION", Value: "ACCEPT"}, - {Name: "FELIX_HEALTHENABLED", Value: "true"}, - {Name: "FELIX_HEALTHPORT", Value: "9099"}, - { - Name: "NODENAME", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}, - }, - }, - { - Name: "NAMESPACE", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}, - }, - }, - {Name: "FELIX_TYPHAK8SNAMESPACE", Value: "calico-system"}, - {Name: "FELIX_TYPHAK8SSERVICENAME", Value: "calico-typha"}, - {Name: "FELIX_TYPHACAFILE", Value: certificatemanagement.TrustedCertBundleMountPath}, - {Name: "FELIX_TYPHACERTFILE", Value: "/node-certs/tls.crt"}, - {Name: "FELIX_TYPHACN", Value: "typha-server"}, - {Name: "FELIX_TYPHAKEYFILE", Value: "/node-certs/tls.key"}, - // Tigera-specific envvars - {Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, - {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: "9081"}, - {Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDELABELS", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDEPOLICIES", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDESERVICE", Value: "true"}, - {Name: "FELIX_FLOWLOGSENABLENETWORKSETS", Value: "true"}, - {Name: "FELIX_FLOWLOGSCOLLECTPROCESSINFO", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEPERNODELIMIT", Value: "1000"}, - {Name: "MULTI_INTERFACE_MODE", Value: operatorv1.MultiInterfaceModeNone.Value()}, - {Name: "NO_DEFAULT_POOLS", Value: "true"}, - } - expectedNodeEnv = configureExpectedNodeEnvIPVersions(expectedNodeEnv, defaultInstance, enableIPv4, enableIPv6) - Expect(ds.Spec.Template.Spec.Containers[0].Env).To(ConsistOf(expectedNodeEnv)) - Expect(len(ds.Spec.Template.Spec.Containers[0].Env)).To(Equal(len(expectedNodeEnv))) - - dirMustExist := corev1.HostPathDirectory - bpfVol := corev1.Volume{Name: "bpffs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs/bpf", Type: &dirMustExist}}} - Expect(ds.Spec.Template.Spec.Volumes).To(ContainElement(bpfVol)) - - bpfVolMount := corev1.VolumeMount{MountPath: "/sys/fs/bpf", Name: "bpffs"} - Expect(ds.Spec.Template.Spec.Containers[0].VolumeMounts).To(ContainElement(bpfVolMount)) - - verifyProbesAndLifecycle(ds, false, true) - }) - It("should render all resources when using Calico CNI on EKS", func() { expectedResources := []struct { name string @@ -871,7 +772,7 @@ var _ = Describe("Node rendering tests", func() { {Name: "xtables-lock", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/run/xtables.lock", Type: &fileOrCreate}}}, {Name: "cni-bin-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/opt/cni/bin", Type: &dirOrCreate}}}, {Name: "cni-net-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/etc/cni/net.d"}}}, - {Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico/cni"}}}, + {Name: "var-log-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}}, {Name: "policysync", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/nodeagent", Type: &dirOrCreate}}}, {Name: "sys-fs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs", Type: &dirOrCreate}}}, {Name: "bpffs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs/bpf", Type: &dirMustExist}}}, @@ -909,7 +810,7 @@ var _ = Describe("Node rendering tests", func() { {MountPath: "/var/run/nodeagent", Name: "policysync"}, {MountPath: "/etc/pki/tls/certs", Name: "tigera-ca-bundle", ReadOnly: true}, {MountPath: "/node-certs", Name: render.NodeTLSSecretName, ReadOnly: true}, - {MountPath: "/var/log/calico/cni", Name: "cni-log-dir", ReadOnly: false}, + {MountPath: "/var/log/calico", Name: "var-log-calico"}, {MountPath: "/sys/fs/bpf", Name: "bpffs"}, } Expect(ds.Spec.Template.Spec.Containers[0].VolumeMounts).To(ConsistOf(expectedNodeVolumeMounts)) @@ -919,7 +820,7 @@ var _ = Describe("Node rendering tests", func() { // Verify readiness and liveness probes. - verifyProbesAndLifecycle(ds, false, false) + verifyProbesAndLifecycle(ds, false) }) It("should properly render a configuration using the AmazonVPC CNI plugin", func() { @@ -1008,6 +909,7 @@ var _ = Describe("Node rendering tests", func() { {Name: "lib-modules", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/lib/modules"}}}, {Name: "var-run-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/calico", Type: &dirOrCreate}}}, {Name: "var-lib-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/lib/calico", Type: &dirOrCreate}}}, + {Name: "var-log-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}}, {Name: "xtables-lock", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/run/xtables.lock", Type: &fileOrCreate}}}, {Name: "policysync", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/nodeagent", Type: &dirOrCreate}}}, {Name: "sys-fs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs", Type: &dirOrCreate}}}, @@ -1042,6 +944,7 @@ var _ = Describe("Node rendering tests", func() { {MountPath: "/run/xtables.lock", Name: "xtables-lock"}, {MountPath: "/var/run/calico", Name: "var-run-calico"}, {MountPath: "/var/lib/calico", Name: "var-lib-calico"}, + {MountPath: "/var/log/calico", Name: "var-log-calico"}, {MountPath: "/var/run/nodeagent", Name: "policysync"}, {MountPath: "/etc/pki/tls/certs", Name: "tigera-ca-bundle", ReadOnly: true}, {MountPath: "/node-certs", Name: render.NodeTLSSecretName, ReadOnly: true}, @@ -1053,7 +956,7 @@ var _ = Describe("Node rendering tests", func() { Expect(ds.Spec.Template.Spec.Tolerations).To(ConsistOf(rmeta.TolerateAll)) // Verify readiness and liveness probes. - verifyProbesAndLifecycle(ds, false, false) + verifyProbesAndLifecycle(ds, false) }) It("should return customized CNI directories when specified", func() { @@ -1073,7 +976,7 @@ var _ = Describe("Node rendering tests", func() { expectedVols := []corev1.Volume{ {Name: "cni-bin-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/custom/cni/bin", Type: &dirOrCreate}}}, {Name: "cni-net-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/custom/cni/net.d"}}}, - {Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico/cni"}}}, + {Name: "var-log-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}}, } Expect(ds.Spec.Template.Spec.Volumes).To(ContainElements(expectedVols)) verifyInitContainers(ds, cfg.Installation) @@ -1121,7 +1024,7 @@ var _ = Describe("Node rendering tests", func() { } // Verify readiness and liveness probes. - verifyProbesAndLifecycle(ds, false, false) + verifyProbesAndLifecycle(ds, false) }, Entry("GKE", operatorv1.PluginGKE, operatorv1.IPAMPluginHostLocal, []corev1.EnvVar{ {Name: "FELIX_INTERFACEPREFIX", Value: "gke"}, @@ -1273,7 +1176,7 @@ var _ = Describe("Node rendering tests", func() { {Name: "xtables-lock", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/run/xtables.lock", Type: &fileOrCreate}}}, {Name: "cni-bin-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/opt/cni/bin", Type: &dirOrCreate}}}, {Name: "cni-net-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/etc/cni/net.d"}}}, - {Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico/cni"}}}, + {Name: "var-log-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}}, {Name: "policysync", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/nodeagent", Type: &dirOrCreate}}}, {Name: "sys-fs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs", Type: &dirOrCreate}}}, {Name: "bpffs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs/bpf", Type: &dirMustExist}}}, @@ -1311,7 +1214,7 @@ var _ = Describe("Node rendering tests", func() { {MountPath: "/var/run/nodeagent", Name: "policysync"}, {MountPath: "/etc/pki/tls/certs", Name: "tigera-ca-bundle", ReadOnly: true}, {MountPath: "/node-certs", Name: render.NodeTLSSecretName, ReadOnly: true}, - {MountPath: "/var/log/calico/cni", Name: "cni-log-dir", ReadOnly: false}, + {MountPath: "/var/log/calico", Name: "var-log-calico"}, {MountPath: "/sys/fs/bpf", Name: "bpffs"}, } Expect(ds.Spec.Template.Spec.Containers[0].VolumeMounts).To(ConsistOf(expectedNodeVolumeMounts)) @@ -1320,7 +1223,7 @@ var _ = Describe("Node rendering tests", func() { Expect(ds.Spec.Template.Spec.Tolerations).To(ConsistOf(rmeta.TolerateAll)) // Verify readiness and liveness probes. - verifyProbesAndLifecycle(ds, false, false) + verifyProbesAndLifecycle(ds, false) }) It("should properly render a configuration using the AmazonVPC CNI plugin", func() { @@ -1407,6 +1310,7 @@ var _ = Describe("Node rendering tests", func() { {Name: "lib-modules", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/lib/modules"}}}, {Name: "var-run-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/calico", Type: &dirOrCreate}}}, {Name: "var-lib-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/lib/calico", Type: &dirOrCreate}}}, + {Name: "var-log-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}}, {Name: "xtables-lock", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/run/xtables.lock", Type: &fileOrCreate}}}, {Name: "policysync", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/nodeagent", Type: &dirOrCreate}}}, {Name: "sys-fs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs", Type: &dirOrCreate}}}, @@ -1441,6 +1345,7 @@ var _ = Describe("Node rendering tests", func() { {MountPath: "/run/xtables.lock", Name: "xtables-lock"}, {MountPath: "/var/run/calico", Name: "var-run-calico"}, {MountPath: "/var/lib/calico", Name: "var-lib-calico"}, + {MountPath: "/var/log/calico", Name: "var-log-calico"}, {MountPath: "/var/run/nodeagent", Name: "policysync"}, {MountPath: "/etc/pki/tls/certs", Name: "tigera-ca-bundle", ReadOnly: true}, {MountPath: "/node-certs", Name: render.NodeTLSSecretName, ReadOnly: true}, @@ -1452,7 +1357,7 @@ var _ = Describe("Node rendering tests", func() { Expect(ds.Spec.Template.Spec.Tolerations).To(ConsistOf(rmeta.TolerateAll)) // Verify readiness and liveness probes. - verifyProbesAndLifecycle(ds, false, false) + verifyProbesAndLifecycle(ds, false) }) It("should render all resources when running on openshift", func() { @@ -1519,7 +1424,7 @@ var _ = Describe("Node rendering tests", func() { {Name: "xtables-lock", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/run/xtables.lock", Type: &fileOrCreate}}}, {Name: "cni-bin-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/lib/cni/bin", Type: &dirOrCreate}}}, {Name: "cni-net-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/multus/cni/net.d"}}}, - {Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico/cni"}}}, + {Name: "var-log-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}}, {Name: "policysync", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/nodeagent", Type: &dirOrCreate}}}, {Name: "flexvol-driver-host", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/etc/kubernetes/kubelet-plugins/volume/exec/nodeagent~uds", Type: &dirOrCreate}}}, {Name: "sys-fs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs", Type: &dirOrCreate}}}, @@ -1582,209 +1487,7 @@ var _ = Describe("Node rendering tests", func() { Expect(ds.Spec.Template.Spec.Containers[0].Env).To(ConsistOf(expectedNodeEnv)) Expect(len(ds.Spec.Template.Spec.Containers[0].Env)).To(Equal(len(expectedNodeEnv))) - verifyProbesAndLifecycle(ds, true, false) - }) - - It("should render all resources when variant is CalicoEnterprise and running on openshift", func() { - expectedResources := []struct { - name string - ns string - group string - version string - kind string - }{ - {name: "calico-node", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ServiceAccount"}, - {name: "calico-node", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, - {name: "calico-node", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: "calico-cni-plugin", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ServiceAccount"}, - {name: "calico-cni-plugin", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, - {name: "calico-cni-plugin", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: "cni-config", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ConfigMap"}, - {name: common.NodeDaemonSetName, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "DaemonSet"}, - } - - defaultInstance.Variant = operatorv1.CalicoEnterprise - defaultInstance.KubernetesProvider = operatorv1.ProviderOpenShift - defaultCNIConfDir, defaultCNIBinDir := render.DefaultCNIDirectories(defaultInstance.KubernetesProvider) - defaultInstance.CNI.ConfDir, defaultInstance.CNI.BinDir = &defaultCNIConfDir, &defaultCNIBinDir - cfg.NodeReporterMetricsPort = 9081 - cfg.FelixHealthPort = 9199 - - component := render.Node(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - Expect(len(resources)).To(Equal(len(expectedResources))) - - // Should render the correct resources. - i := 0 - for _, expectedRes := range expectedResources { - rtest.ExpectResourceTypeAndObjectMetadata(resources[i], expectedRes.name, expectedRes.ns, expectedRes.group, expectedRes.version, expectedRes.kind) - i++ - } - - // calico-node clusterRole should have openshift securitycontextconstraints PolicyRule - nodeRole := rtest.GetResource(resources, "calico-node", "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - Expect(nodeRole.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{"security.openshift.io"}, - Resources: []string{"securitycontextconstraints"}, - Verbs: []string{"use"}, - ResourceNames: []string{"privileged"}, - })) - - // The DaemonSet should have the correct configuration. - ds := rtest.GetResource(resources, "calico-node", "calico-system", "apps", "v1", "DaemonSet").(*appsv1.DaemonSet) - Expect(ds.Spec.Template.Spec.Containers[0].Image).To(Equal(components.TigeraRegistry + "tigera/node:" + components.ComponentTigeraNode.Version)) - - // The pod template should have node critical priority - Expect(ds.Spec.Template.Spec.PriorityClassName).To(Equal(render.NodePriorityClassName)) - - verifyInitContainers(ds, defaultInstance) - expectedNodeEnv := []corev1.EnvVar{ - // Default envvars. - {Name: "DATASTORE_TYPE", Value: "kubernetes"}, - {Name: "WAIT_FOR_DATASTORE", Value: "true"}, - {Name: "CALICO_MANAGE_CNI", Value: "true"}, - {Name: "CALICO_NETWORKING_BACKEND", Value: "bird"}, - {Name: "CLUSTER_TYPE", Value: "k8s,operator,openshift,bgp"}, - {Name: "CALICO_DISABLE_FILE_LOGGING", Value: "false"}, - {Name: "FELIX_DEFAULTENDPOINTTOHOSTACTION", Value: "ACCEPT"}, - {Name: "FELIX_HEALTHENABLED", Value: "true"}, - {Name: "FELIX_HEALTHPORT", Value: "9199"}, - { - Name: "NODENAME", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}, - }, - }, - { - Name: "NAMESPACE", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}, - }, - }, - {Name: "FELIX_TYPHAK8SNAMESPACE", Value: "calico-system"}, - {Name: "FELIX_TYPHAK8SSERVICENAME", Value: "calico-typha"}, - {Name: "FELIX_TYPHACAFILE", Value: certificatemanagement.TrustedCertBundleMountPath}, - {Name: "FELIX_TYPHACERTFILE", Value: "/node-certs/tls.crt"}, - {Name: "FELIX_TYPHACN", Value: "typha-server"}, - {Name: "FELIX_TYPHAKEYFILE", Value: "/node-certs/tls.key"}, - // Tigera-specific envvars - {Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, - {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: "9081"}, - {Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDELABELS", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDEPOLICIES", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDESERVICE", Value: "true"}, - {Name: "FELIX_FLOWLOGSENABLENETWORKSETS", Value: "true"}, - {Name: "FELIX_FLOWLOGSCOLLECTPROCESSINFO", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEPERNODELIMIT", Value: "1000"}, - {Name: "MULTI_INTERFACE_MODE", Value: operatorv1.MultiInterfaceModeNone.Value()}, - {Name: "NO_DEFAULT_POOLS", Value: "true"}, - } - expectedNodeEnv = configureExpectedNodeEnvIPVersions(expectedNodeEnv, defaultInstance, enableIPv4, enableIPv6) - Expect(ds.Spec.Template.Spec.Containers[0].Env).To(ConsistOf(expectedNodeEnv)) - Expect(len(ds.Spec.Template.Spec.Containers[0].Env)).To(Equal(len(expectedNodeEnv))) - - verifyProbesAndLifecycle(ds, true, true) - }) - - It("should render all resources when variant is CalicoEnterprise and running on RKE2", func() { - expectedResources := []struct { - name string - ns string - group string - version string - kind string - }{ - {name: "calico-node", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ServiceAccount"}, - {name: "calico-node", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, - {name: "calico-node", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: "calico-cni-plugin", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ServiceAccount"}, - {name: "calico-cni-plugin", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, - {name: "calico-cni-plugin", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: "cni-config", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ConfigMap"}, - {name: common.NodeDaemonSetName, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "DaemonSet"}, - } - - defaultInstance.Variant = operatorv1.CalicoEnterprise - defaultInstance.KubernetesProvider = operatorv1.ProviderRKE2 - defaultCNIConfDir, defaultCNIBinDir := render.DefaultCNIDirectories(defaultInstance.KubernetesProvider) - defaultInstance.CNI.ConfDir, defaultInstance.CNI.BinDir = &defaultCNIConfDir, &defaultCNIBinDir - cfg.NodeReporterMetricsPort = 9081 - cfg.FelixHealthPort = 9199 - - component := render.Node(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - Expect(len(resources)).To(Equal(len(expectedResources)), fmt.Sprintf("Actual resources: %#v", resources)) - - // Should render the correct resources. - i := 0 - for _, expectedRes := range expectedResources { - rtest.ExpectResourceTypeAndObjectMetadata(resources[i], expectedRes.name, expectedRes.ns, expectedRes.group, expectedRes.version, expectedRes.kind) - i++ - } - - // The DaemonSet should have the correct configuration. - ds := rtest.GetResource(resources, "calico-node", "calico-system", "apps", "v1", "DaemonSet").(*appsv1.DaemonSet) - Expect(ds.Spec.Template.Spec.Containers[0].Image).To(Equal(components.TigeraRegistry + "tigera/node:" + components.ComponentTigeraNode.Version)) - - // The pod template should have node critical priority - Expect(ds.Spec.Template.Spec.PriorityClassName).To(Equal(render.NodePriorityClassName)) - - verifyInitContainers(ds, defaultInstance) - - expectedNodeEnv := []corev1.EnvVar{ - // Default envvars. - {Name: "DATASTORE_TYPE", Value: "kubernetes"}, - {Name: "WAIT_FOR_DATASTORE", Value: "true"}, - {Name: "CALICO_MANAGE_CNI", Value: "true"}, - {Name: "CALICO_NETWORKING_BACKEND", Value: "bird"}, - {Name: "CLUSTER_TYPE", Value: "k8s,operator,bgp"}, - {Name: "CALICO_DISABLE_FILE_LOGGING", Value: "false"}, - {Name: "FELIX_DEFAULTENDPOINTTOHOSTACTION", Value: "ACCEPT"}, - {Name: "FELIX_HEALTHENABLED", Value: "true"}, - {Name: "FELIX_HEALTHPORT", Value: "9199"}, - { - Name: "NODENAME", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}, - }, - }, - { - Name: "NAMESPACE", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}, - }, - }, - {Name: "FELIX_TYPHAK8SNAMESPACE", Value: "calico-system"}, - {Name: "FELIX_TYPHAK8SSERVICENAME", Value: "calico-typha"}, - {Name: "FELIX_TYPHACAFILE", Value: certificatemanagement.TrustedCertBundleMountPath}, - {Name: "FELIX_TYPHACERTFILE", Value: "/node-certs/tls.crt"}, - {Name: "FELIX_TYPHACN", Value: "typha-server"}, - {Name: "FELIX_TYPHAKEYFILE", Value: "/node-certs/tls.key"}, - {Name: "NO_DEFAULT_POOLS", Value: "true"}, - // Tigera-specific envvars - {Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, - {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: "9081"}, - {Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDELABELS", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDEPOLICIES", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDESERVICE", Value: "true"}, - {Name: "FELIX_FLOWLOGSENABLENETWORKSETS", Value: "true"}, - {Name: "FELIX_FLOWLOGSCOLLECTPROCESSINFO", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEPERNODELIMIT", Value: "1000"}, - - // The RKE2 envvar overrides. - {Name: "MULTI_INTERFACE_MODE", Value: operatorv1.MultiInterfaceModeNone.Value()}, - } - expectedNodeEnv = configureExpectedNodeEnvIPVersions(expectedNodeEnv, defaultInstance, enableIPv4, enableIPv6) - Expect(ds.Spec.Template.Spec.Containers[0].Env).To(ConsistOf(expectedNodeEnv)) - Expect(len(ds.Spec.Template.Spec.Containers[0].Env)).To(Equal(len(expectedNodeEnv))) - - verifyProbesAndLifecycle(ds, true, true) + verifyProbesAndLifecycle(ds, true) }) It("should render volumes and node volumemounts when bird templates are provided", func() { @@ -2043,7 +1746,6 @@ var _ = Describe("Node rendering tests", func() { It("should not enable prometheus metrics if NodeMetricsPort is nil", func() { defaultInstance.Variant = operatorv1.CalicoEnterprise defaultInstance.NodeMetricsPort = nil - cfg.NodeReporterMetricsPort = 9081 component := render.Node(&cfg) Expect(component.ResolveImages(nil)).To(BeNil()) @@ -2057,7 +1759,8 @@ var _ = Describe("Node rendering tests", func() { ds := dsResource.(*appsv1.DaemonSet) Expect(ds.Spec.Template.Spec.Containers[0].Env).ToNot(ContainElement(notExpectedEnvVar)) - // It should have the reporter port, though. + // The reporter port env is added by the enterprise node modifier, not the + // base render, so it should be absent here. expected := corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERPORT"} Expect(ds.Spec.Template.Spec.Containers[0].Env).ToNot(ContainElement(expected)) }) @@ -2869,7 +2572,7 @@ var _ = Describe("Node rendering tests", func() { Expect(ds.Spec.Template.Spec.Containers[0].Env).To(ConsistOf(expectedNodeEnv)) // Verify readiness and liveness probes. - verifyProbesAndLifecycle(ds, false, false) + verifyProbesAndLifecycle(ds, false) }) DescribeTable("test node probes", @@ -2894,7 +2597,7 @@ var _ = Describe("Node rendering tests", func() { Expect(dsResource).ToNot(BeNil()) ds := dsResource.(*appsv1.DaemonSet) - verifyProbesAndLifecycle(ds, isOpenshift, isEnterprise) + verifyProbesAndLifecycle(ds, isOpenshift) }, Entry("k8s Calico OS no BGP", false, false, operatorv1.BGPDisabled), @@ -3212,7 +2915,7 @@ var _ = Describe("Node rendering tests", func() { }) // verifyProbesAndLifecycle asserts the expected node liveness and readiness probe plus pod lifecycle settings. -func verifyProbesAndLifecycle(ds *appsv1.DaemonSet, isOpenshift, isEnterprise bool) { +func verifyProbesAndLifecycle(ds *appsv1.DaemonSet, isOpenshift bool) { // Verify readiness and liveness probes. expectedReadiness := &corev1.Probe{ PeriodSeconds: 10, @@ -3246,14 +2949,14 @@ func verifyProbesAndLifecycle(ds *appsv1.DaemonSet, isOpenshift, isEnterprise bo } ExpectWithOffset(1, found).To(BeTrue()) + // The base render produces the same readiness command for all variants; the + // enterprise --bgp-metrics-ready check is added by the node modifier and is + // covered in the enterprise package tests. var expectedReadinessCmd []string - switch { - case !bgp: - expectedReadinessCmd = []string{"/usr/bin/calico", "component", "node", "health", "--felix-ready"} - case bgp && isEnterprise: - expectedReadinessCmd = []string{"/usr/bin/calico", "component", "node", "health", "--bird-ready", "--felix-ready", "--bgp-metrics-ready"} - case bgp: + if bgp { expectedReadinessCmd = []string{"/usr/bin/calico", "component", "node", "health", "--bird-ready", "--felix-ready"} + } else { + expectedReadinessCmd = []string{"/usr/bin/calico", "component", "node", "health", "--felix-ready"} } expectedReadiness.ProbeHandler = corev1.ProbeHandler{Exec: &corev1.ExecAction{Command: expectedReadinessCmd}} diff --git a/pkg/render/render_test.go b/pkg/render/render_test.go index 42afa04570..a172ce4f00 100644 --- a/pkg/render/render_test.go +++ b/pkg/render/render_test.go @@ -79,17 +79,16 @@ func allCalicoComponents( secretsAndConfigMaps := render.NewCreationPassthrough(objs...) nodeCfg := &render.NodeConfiguration{ - K8sServiceEp: k8sServiceEp, - Installation: cr, - TLS: typhaNodeTLS, - NodeAppArmorProfile: nodeAppArmorProfile, - ClusterDomain: clusterDomain, - NodeReporterMetricsPort: nodeReporterMetricsPort, - BGPLayouts: bgpLayout, - LogCollector: logCollector, - BirdTemplates: bt, - MigrateNamespaces: up, - FelixHealthPort: 9099, + K8sServiceEp: k8sServiceEp, + Installation: cr, + TLS: typhaNodeTLS, + NodeAppArmorProfile: nodeAppArmorProfile, + ClusterDomain: clusterDomain, + BGPLayouts: bgpLayout, + LogCollector: logCollector, + BirdTemplates: bt, + MigrateNamespaces: up, + FelixHealthPort: 9099, } typhaCfg := &render.TyphaConfiguration{ K8sServiceEp: k8sServiceEp, From 311b79ef731b80b59c88d118eb939593c41ad456 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Tue, 9 Jun 2026 09:55:01 -0700 Subject: [PATCH 16/38] Simplify render context extension entrypoints Drop the functional-options builder for Inputs (one call site, all fields always set) in favor of a plain struct literal, and replace the single-method RenderContextFactory interface with a registered builder func. All three extension seams now register a func. --- .../installation/core_controller.go | 20 ++-- pkg/enterprise/installation.go | 15 ++- pkg/enterprise/installation_test.go | 32 +++---- pkg/extensions/factory.go | 91 +++++-------------- pkg/extensions/factory_test.go | 44 +++++---- pkg/extensions/modifier.go | 4 +- 6 files changed, 78 insertions(+), 128 deletions(-) diff --git a/pkg/controller/installation/core_controller.go b/pkg/controller/installation/core_controller.go index bbeb8582af..584c27e087 100644 --- a/pkg/controller/installation/core_controller.go +++ b/pkg/controller/installation/core_controller.go @@ -1214,19 +1214,19 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile } // Build the render context handed to registered modifiers. The core operator - // factory returns just the base context; an extension's factory additionally + // builder returns just the base context; an extension's builder additionally // does controller-side work for its variant - validating config and creating // the node-prometheus certificate, adding it (and the prometheus/esgw certs) // to the trusted bundle - and may abort the reconcile by returning an error. - modCtx, err := extensions.GetRenderContextFactory().New( - extensions.WithContext(ctx), - extensions.WithClient(r.client), - extensions.WithInstallation(&instance.Spec), - extensions.WithFelixConfiguration(felixConfiguration), - extensions.WithCertificateManager(certificateManager), - extensions.WithTrustedBundle(typhaNodeTLS.TrustedBundle), - extensions.WithClusterDomain(r.clusterDomain), - ) + modCtx, err := extensions.BuildRenderContext(extensions.Inputs{ + Ctx: ctx, + Client: r.client, + Installation: &instance.Spec, + FelixConfiguration: felixConfiguration, + CertificateManager: certificateManager, + TrustedBundle: typhaNodeTLS.TrustedBundle, + ClusterDomain: r.clusterDomain, + }) if err != nil { r.status.SetDegraded(operatorv1.ResourceCreateError, "Error preparing installation extension", err, reqLogger) return reconcile.Result{}, err diff --git a/pkg/enterprise/installation.go b/pkg/enterprise/installation.go index 596229a635..98fba501ae 100644 --- a/pkg/enterprise/installation.go +++ b/pkg/enterprise/installation.go @@ -26,18 +26,15 @@ import ( "github.com/tigera/operator/pkg/render/monitor" ) -// installationFactory is the Calico Enterprise RenderContextFactory. It builds -// the base render context and then does the controller-side work the modifiers -// can't: validating config and creating/fetching the certificates that feed the -// trusted bundle. -type installationFactory struct{} - func registerInstallation() { - extensions.RegisterRenderContextFactory(&installationFactory{}) + extensions.RegisterRenderContextBuilder(buildRenderContext) } -func (f *installationFactory) New(opts ...extensions.RenderContextOption) (extensions.RenderContext, error) { - in := extensions.ApplyInputs(opts...) +// buildRenderContext is the Calico Enterprise RenderContextBuilder. It builds +// the base render context and then does the controller-side work the modifiers +// can't: validating config and creating/fetching the certificates that feed the +// trusted bundle. +func buildRenderContext(in extensions.Inputs) (extensions.RenderContext, error) { rc := extensions.BaseRenderContext(in) if in.Installation == nil || !in.Installation.Variant.IsEnterprise() { return rc, nil diff --git a/pkg/enterprise/installation_test.go b/pkg/enterprise/installation_test.go index 18c38c312a..f3aa09f887 100644 --- a/pkg/enterprise/installation_test.go +++ b/pkg/enterprise/installation_test.go @@ -32,34 +32,34 @@ import ( "github.com/tigera/operator/pkg/extensions" ) -var _ = Describe("installation render context factory", func() { +var _ = Describe("installation render context builder", func() { BeforeEach(func() { enterprise.Register() }) AfterEach(func() { extensions.ResetForTest() }) It("rejects a zero prometheus reporter port", func() { port := 0 - opts := newOpts(operatorv1.CalicoEnterprise) - opts = append(opts, extensions.WithFelixConfiguration(&v3.FelixConfiguration{ + in := newInputs(operatorv1.CalicoEnterprise) + in.FelixConfiguration = &v3.FelixConfiguration{ Spec: v3.FelixConfigurationSpec{PrometheusReporterPort: &port}, - })) - _, err := extensions.GetRenderContextFactory().New(opts...) + } + _, err := extensions.BuildRenderContext(in) Expect(err).To(HaveOccurred()) }) It("creates the node prometheus keypair for the enterprise variant", func() { - rc, err := extensions.GetRenderContextFactory().New(newOpts(operatorv1.CalicoEnterprise)...) + rc, err := extensions.BuildRenderContext(newInputs(operatorv1.CalicoEnterprise)) Expect(err).NotTo(HaveOccurred()) Expect(rc.NodePrometheusTLS).NotTo(BeNil()) }) It("is a no-op for the Calico variant", func() { - rc, err := extensions.GetRenderContextFactory().New(newOpts(operatorv1.Calico)...) + rc, err := extensions.BuildRenderContext(newInputs(operatorv1.Calico)) Expect(err).NotTo(HaveOccurred()) Expect(rc.NodePrometheusTLS).To(BeNil()) }) }) -func newOpts(variant operatorv1.ProductVariant) []extensions.RenderContextOption { +func newInputs(variant operatorv1.ProductVariant) extensions.Inputs { scheme := runtime.NewScheme() Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) c := ctrlrfake.DefaultFakeClientBuilder(scheme).Build() @@ -68,13 +68,13 @@ func newOpts(variant operatorv1.ProductVariant) []extensions.RenderContextOption Expect(err).NotTo(HaveOccurred()) trustedBundle := certManager.CreateTrustedBundle() - return []extensions.RenderContextOption{ - extensions.WithContext(context.Background()), - extensions.WithClient(c), - extensions.WithInstallation(&operatorv1.InstallationSpec{Variant: variant}), - extensions.WithFelixConfiguration(&v3.FelixConfiguration{}), - extensions.WithCertificateManager(certManager), - extensions.WithTrustedBundle(trustedBundle), - extensions.WithClusterDomain("cluster.local"), + return extensions.Inputs{ + Ctx: context.Background(), + Client: c, + Installation: &operatorv1.InstallationSpec{Variant: variant}, + FelixConfiguration: &v3.FelixConfiguration{}, + CertificateManager: certManager, + TrustedBundle: trustedBundle, + ClusterDomain: "cluster.local", } } diff --git a/pkg/extensions/factory.go b/pkg/extensions/factory.go index f6081635ca..bf2d46f246 100644 --- a/pkg/extensions/factory.go +++ b/pkg/extensions/factory.go @@ -25,11 +25,11 @@ import ( "github.com/tigera/operator/pkg/tls/certificatemanagement" ) -// Inputs is the reconcile state a RenderContextFactory builds a RenderContext -// from. The installation controller populates it through the With* options. It -// carries both the values that flow straight into the RenderContext and the -// side-effecting dependencies (Client, CertificateManager) a factory needs to -// produce controller-side artifacts. +// Inputs is the reconcile state a RenderContextBuilder builds a RenderContext +// from. The installation controller populates it directly. It carries both the +// values that flow straight into the RenderContext and the side-effecting +// dependencies (Client, CertificateManager) a builder needs to produce +// controller-side artifacts. type Inputs struct { Ctx context.Context Client client.Client @@ -40,62 +40,19 @@ type Inputs struct { ClusterDomain string } -// RenderContextOption sets a field on the Inputs a factory builds from. -type RenderContextOption func(*Inputs) - -func WithContext(ctx context.Context) RenderContextOption { - return func(in *Inputs) { in.Ctx = ctx } -} - -func WithClient(c client.Client) RenderContextOption { - return func(in *Inputs) { in.Client = c } -} - -func WithInstallation(i *operatorv1.InstallationSpec) RenderContextOption { - return func(in *Inputs) { in.Installation = i } -} - -func WithFelixConfiguration(fc *v3.FelixConfiguration) RenderContextOption { - return func(in *Inputs) { in.FelixConfiguration = fc } -} - -func WithCertificateManager(cm certificatemanager.CertificateManager) RenderContextOption { - return func(in *Inputs) { in.CertificateManager = cm } -} - -func WithTrustedBundle(tb certificatemanagement.TrustedBundle) RenderContextOption { - return func(in *Inputs) { in.TrustedBundle = tb } -} - -func WithClusterDomain(d string) RenderContextOption { - return func(in *Inputs) { in.ClusterDomain = d } -} - -// RenderContextFactory builds the RenderContext handed to render modifiers. New -// applies the options, performs any controller-side work (creating -// certificates, extending the trusted bundle), and returns the assembled -// RenderContext - or an error that aborts the reconcile. +// RenderContextBuilder builds the RenderContext handed to render modifiers. It +// performs any controller-side work (creating certificates, extending the +// trusted bundle) and returns the assembled RenderContext - or an error that +// aborts the reconcile. // // This is the generic seam controllers use to extend base operator behavior; // its first consumer is Calico Enterprise, but nothing here is enterprise // specific. The core operator default does no side-effecting work. -type RenderContextFactory interface { - New(opts ...RenderContextOption) (RenderContext, error) -} - -// ApplyInputs returns the Inputs produced by applying opts. Factories use it to -// collect the option-supplied state before doing their work. -func ApplyInputs(opts ...RenderContextOption) Inputs { - var in Inputs - for _, o := range opts { - o(&in) - } - return in -} +type RenderContextBuilder func(in Inputs) (RenderContext, error) // BaseRenderContext maps the generically-gathered inputs onto a RenderContext. -// The default factory and every registered factory build on it, so the base -// fields are assembled in exactly one place. A factory layers its side-effect +// The default builder and every registered builder build on it, so the base +// fields are assembled in exactly one place. A builder layers its side-effect // artifacts (e.g. NodePrometheusTLS) on top of the returned value. func BaseRenderContext(in Inputs) RenderContext { return RenderContext{ @@ -106,22 +63,20 @@ func BaseRenderContext(in Inputs) RenderContext { } } -// defaultFactory is the core operator's RenderContextFactory. It does no +// defaultRenderContextBuilder is the core operator's builder. It does no // side-effecting work and returns just the base RenderContext. -type defaultFactory struct{} - -func (defaultFactory) New(opts ...RenderContextOption) (RenderContext, error) { - return BaseRenderContext(ApplyInputs(opts...)), nil +func defaultRenderContextBuilder(in Inputs) (RenderContext, error) { + return BaseRenderContext(in), nil } -var renderContextFactory RenderContextFactory = defaultFactory{} +var renderContextBuilder RenderContextBuilder = defaultRenderContextBuilder -// RegisterRenderContextFactory installs f as the factory the installation -// controller uses. Registration replaces any prior factory, so it is safe to -// call more than once. Without a registered factory the core operator default +// RegisterRenderContextBuilder installs f as the builder the installation +// controller uses. Registration replaces any prior builder, so it is safe to +// call more than once. Without a registered builder the core operator default // applies. -func RegisterRenderContextFactory(f RenderContextFactory) { renderContextFactory = f } +func RegisterRenderContextBuilder(f RenderContextBuilder) { renderContextBuilder = f } -// GetRenderContextFactory returns the registered factory, or the core operator -// default when none is registered. -func GetRenderContextFactory() RenderContextFactory { return renderContextFactory } +// BuildRenderContext builds the RenderContext from in using the registered +// builder, or the core operator default when none is registered. +func BuildRenderContext(in Inputs) (RenderContext, error) { return renderContextBuilder(in) } diff --git a/pkg/extensions/factory_test.go b/pkg/extensions/factory_test.go index 408704b7ad..7954f402bd 100644 --- a/pkg/extensions/factory_test.go +++ b/pkg/extensions/factory_test.go @@ -24,50 +24,48 @@ import ( "github.com/tigera/operator/pkg/extensions" ) -var _ = Describe("render context factory", func() { +var _ = Describe("render context builder", func() { AfterEach(func() { extensions.ResetForTest() }) - It("returns the base render context from the default factory", func() { + It("returns the base render context from the default builder", func() { install := &operatorv1.InstallationSpec{Variant: operatorv1.Calico} - rc, err := extensions.GetRenderContextFactory().New( - extensions.WithInstallation(install), - extensions.WithClusterDomain("cluster.local"), - ) + rc, err := extensions.BuildRenderContext(extensions.Inputs{ + Installation: install, + ClusterDomain: "cluster.local", + }) Expect(err).NotTo(HaveOccurred()) Expect(rc.Installation).To(BeIdenticalTo(install)) Expect(rc.ClusterDomain).To(Equal("cluster.local")) Expect(rc.NodePrometheusTLS).To(BeNil()) }) - It("uses a registered factory in place of the default", func() { - extensions.RegisterRenderContextFactory(&fakeFactory{}) - rc, err := extensions.GetRenderContextFactory().New() + It("uses a registered builder in place of the default", func() { + extensions.RegisterRenderContextBuilder(fakeBuilder(nil)) + rc, err := extensions.BuildRenderContext(extensions.Inputs{}) Expect(err).NotTo(HaveOccurred()) Expect(rc.ClusterDomain).To(Equal("from-fake")) }) - It("surfaces the factory error", func() { - extensions.RegisterRenderContextFactory(&fakeFactory{err: errors.New("boom")}) - _, err := extensions.GetRenderContextFactory().New() + It("surfaces the builder error", func() { + extensions.RegisterRenderContextBuilder(fakeBuilder(errors.New("boom"))) + _, err := extensions.BuildRenderContext(extensions.Inputs{}) Expect(err).To(MatchError("boom")) }) - It("restores the default factory on reset", func() { - extensions.RegisterRenderContextFactory(&fakeFactory{}) + It("restores the default builder on reset", func() { + extensions.RegisterRenderContextBuilder(fakeBuilder(nil)) extensions.ResetForTest() - rc, err := extensions.GetRenderContextFactory().New(extensions.WithClusterDomain("real")) + rc, err := extensions.BuildRenderContext(extensions.Inputs{ClusterDomain: "real"}) Expect(err).NotTo(HaveOccurred()) Expect(rc.ClusterDomain).To(Equal("real")) }) }) -type fakeFactory struct { - err error -} - -func (f *fakeFactory) New(_ ...extensions.RenderContextOption) (extensions.RenderContext, error) { - if f.err != nil { - return extensions.RenderContext{}, f.err +func fakeBuilder(err error) extensions.RenderContextBuilder { + return func(_ extensions.Inputs) (extensions.RenderContext, error) { + if err != nil { + return extensions.RenderContext{}, err + } + return extensions.RenderContext{ClusterDomain: "from-fake"}, nil } - return extensions.RenderContext{ClusterDomain: "from-fake"}, nil } diff --git a/pkg/extensions/modifier.go b/pkg/extensions/modifier.go index 5b911edc72..b872638531 100644 --- a/pkg/extensions/modifier.go +++ b/pkg/extensions/modifier.go @@ -57,9 +57,9 @@ func FindObject[T client.Object](objs []client.Object, name string) (T, bool) { } // ResetForTest clears every registry: modifiers, image overrides, and the -// render context factory. Test-only. +// render context builder. Test-only. func ResetForTest() { modifiers = map[string]Modifier{} - renderContextFactory = defaultFactory{} + renderContextBuilder = defaultRenderContextBuilder imageoverride.ResetForTest() } From 5a7cad95955cb9a0f2016e4dcac8b6a1c554351d Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Tue, 9 Jun 2026 10:27:37 -0700 Subject: [PATCH 17/38] Variant-scope extension registration and tighten the seams Register modifiers, image overrides, and the render context builder per variant. The registries now gate on the installation variant, so the enterprise funcs drop their self-gate guards (the IsEnterprise checks the PR set out to remove) and the image override drops its decline bool - it only runs for its own variant. Move the node prometheus reporter keypair mounting (volume, mount, cert-management init container, pod hash annotation) into the node modifier, and remove NodeConfiguration.PrometheusServerTLS along with the round-trip through the installation controller. Core node render no longer carries a prometheus mount; in calico the keypair is never created. Rename Extensible.Name() to ModifierKey() so an unrelated Name() method can't make a component modifier-eligible by accident. --- .../installation/core_controller.go | 1 - pkg/controller/utils/component.go | 4 +- pkg/controller/utils/component_test.go | 7 ++- pkg/enterprise/installation.go | 6 +- pkg/enterprise/node.go | 56 ++++++++++++++----- pkg/enterprise/typha.go | 7 +-- pkg/extensions/factory.go | 42 +++++++------- pkg/extensions/factory_test.go | 34 ++++++++--- pkg/extensions/image.go | 14 ++--- pkg/extensions/image_test.go | 15 ++--- pkg/extensions/modifier.go | 39 ++++++++----- pkg/extensions/modifier_test.go | 34 ++++++++--- pkg/extensions/rendercontext.go | 9 ++- pkg/imageoverride/imageoverride.go | 35 +++++++----- pkg/render/component.go | 14 ++--- pkg/render/node.go | 18 +----- pkg/render/node_enterprise_test.go | 19 ++++--- pkg/render/typha.go | 2 +- 18 files changed, 210 insertions(+), 146 deletions(-) diff --git a/pkg/controller/installation/core_controller.go b/pkg/controller/installation/core_controller.go index 584c27e087..568a59f954 100644 --- a/pkg/controller/installation/core_controller.go +++ b/pkg/controller/installation/core_controller.go @@ -1521,7 +1521,6 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile NodeAppArmorProfile: nodeAppArmorProfile, MigrateNamespaces: needsNamespaceMigration, CanRemoveCNIFinalizer: canRemoveCNI, - PrometheusServerTLS: nodePrometheusTLS, FelixHealthPort: *felixConfiguration.Spec.HealthPort, NodeCgroupV2Path: felixConfiguration.Spec.CgroupV2Path, V3CRDs: r.v3CRDs, diff --git a/pkg/controller/utils/component.go b/pkg/controller/utils/component.go index 3e343c4540..57238556da 100644 --- a/pkg/controller/utils/component.go +++ b/pkg/controller/utils/component.go @@ -468,8 +468,8 @@ func (c *componentHandler) CreateOrUpdateOrDelete(ctx context.Context, component var cronJobs []types.NamespacedName objsToCreate, objsToDelete := component.Objects() - if named, ok := component.(render.Extensible); ok { - objsToCreate = extensions.ApplyModifiers(named.Name(), c.modCtx, objsToCreate) + if ext, ok := component.(render.Extensible); ok { + objsToCreate = extensions.ApplyModifiers(ext.ModifierKey(), c.modCtx, objsToCreate) } // Load the InstallationSpec once and reuse it for every object: createOrUpdateObject needs it diff --git a/pkg/controller/utils/component_test.go b/pkg/controller/utils/component_test.go index 3081adfe19..eeb989e501 100644 --- a/pkg/controller/utils/component_test.go +++ b/pkg/controller/utils/component_test.go @@ -2493,7 +2493,7 @@ var _ = Describe("componentHandler modifier application", func() { }) It("applies registered modifiers to a named component before create", func() { - extensions.Modify("fake", func(ctx extensions.RenderContext, objs []client.Object) []client.Object { + extensions.Modify(operatorv1.CalicoEnterprise, "fake", func(ctx extensions.RenderContext, objs []client.Object) []client.Object { cm := objs[0].(*corev1.ConfigMap) cm.Data = map[string]string{"patched": "yes"} return objs @@ -2504,7 +2504,8 @@ var _ = Describe("componentHandler modifier application", func() { Expect(corev1.SchemeBuilder.AddToScheme(s)).NotTo(HaveOccurred()) c := ctrlrfake.DefaultFakeClientBuilder(s).Build() - handler := NewComponentHandler(logf.Log, c, s, nil) + modCtx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}} + handler := NewComponentHandler(logf.Log, c, s, nil, WithRenderContext(modCtx)) comp := &namedFakeComponent{name: "fake", obj: &corev1.ConfigMap{ TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: "cm", Namespace: "default"}, @@ -2523,7 +2524,7 @@ type namedFakeComponent struct { obj client.Object } -func (f *namedFakeComponent) Name() string { return f.name } +func (f *namedFakeComponent) ModifierKey() string { return f.name } func (f *namedFakeComponent) ResolveImages(*operatorv1.ImageSet) error { return nil } func (f *namedFakeComponent) Objects() ([]client.Object, []client.Object) { return []client.Object{f.obj}, nil diff --git a/pkg/enterprise/installation.go b/pkg/enterprise/installation.go index 98fba501ae..cd49480c68 100644 --- a/pkg/enterprise/installation.go +++ b/pkg/enterprise/installation.go @@ -18,6 +18,7 @@ import ( "errors" "fmt" + operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/dns" "github.com/tigera/operator/pkg/extensions" @@ -27,7 +28,7 @@ import ( ) func registerInstallation() { - extensions.RegisterRenderContextBuilder(buildRenderContext) + extensions.RegisterRenderContextBuilder(operatorv1.CalicoEnterprise, buildRenderContext) } // buildRenderContext is the Calico Enterprise RenderContextBuilder. It builds @@ -36,9 +37,6 @@ func registerInstallation() { // trusted bundle. func buildRenderContext(in extensions.Inputs) (extensions.RenderContext, error) { rc := extensions.BaseRenderContext(in) - if in.Installation == nil || !in.Installation.Variant.IsEnterprise() { - return rc, nil - } // Reject the unsupported zero reporter port. The port value itself is derived // in the node modifier; only this validation lives here. diff --git a/pkg/enterprise/node.go b/pkg/enterprise/node.go index 9bf15ece57..455c7b403f 100644 --- a/pkg/enterprise/node.go +++ b/pkg/enterprise/node.go @@ -31,6 +31,7 @@ import ( "github.com/tigera/operator/pkg/controller/utils" "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" + rmeta "github.com/tigera/operator/pkg/render/common/meta" ) const ( @@ -46,13 +47,10 @@ const ( ) func registerNode() { - extensions.OverrideImage(render.ComponentNameNode, func(in *operatorv1.InstallationSpec) (components.Component, bool) { - if !in.Variant.IsEnterprise() { - return components.Component{}, false - } - return components.ComponentTigeraNode, true + extensions.OverrideImage(operatorv1.CalicoEnterprise, render.ComponentNameNode, func(in *operatorv1.InstallationSpec) components.Component { + return components.ComponentTigeraNode }) - extensions.Modify(render.ComponentNameNode, modifyNode) + extensions.Modify(operatorv1.CalicoEnterprise, render.ComponentNameNode, modifyNode) } // modifyNode layers Calico Enterprise behavior onto the rendered calico/node @@ -60,10 +58,6 @@ func registerNode() { // daemonset configuration (flow/DNS log env, prometheus reporter, BGP metrics // readiness check, multi-interface mode, and the calico log volume). func modifyNode(ctx extensions.RenderContext, objs []client.Object) []client.Object { - if ctx.Installation == nil || !ctx.Installation.Variant.IsEnterprise() { - return objs - } - if role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, render.CalicoNodeObjectName); ok { role.Rules = append(role.Rules, nodeEnterpriseRules()...) } @@ -115,9 +109,10 @@ func nodeEnterpriseRules() []rbacv1.PolicyRule { } // modifyNodeDaemonSet applies the Enterprise-specific daemonset changes that the -// base render leaves out: the Enterprise felix env, multi-interface mode, and the -// BGP metrics readiness check. The calico log volume is mounted by the base -// render for both variants, so it is not handled here. +// base render leaves out: the Enterprise felix env, multi-interface mode, the +// BGP metrics readiness check, and the prometheus reporter keypair mount. The +// calico log volume is mounted by the base render for both variants, so it is +// not handled here. func modifyNodeDaemonSet(ctx extensions.RenderContext, ds *appsv1.DaemonSet) { spec := &ds.Spec.Template.Spec @@ -143,6 +138,41 @@ func modifyNodeDaemonSet(ctx extensions.RenderContext, ds *appsv1.DaemonSet) { c.ReadinessProbe.Exec.Command = append(c.ReadinessProbe.Exec.Command, "--bgp-metrics-ready") } } + + mountNodePrometheusTLS(ctx, ds) +} + +// mountNodePrometheusTLS mounts the node prometheus reporter keypair onto the +// daemonset: the volume, the calico-node volume mount, the cert-management init +// container (when in use), and the pod hash annotation that rolls the pods on +// cert rotation. The keypair has cluster side effects, so the enterprise render +// context builder creates it and hands it in via ctx rather than the modifier +// building it. In core (calico) the keypair is never created, so the base node +// render carries no prometheus mount at all. +func mountNodePrometheusTLS(ctx extensions.RenderContext, ds *appsv1.DaemonSet) { + if ctx.NodePrometheusTLS == nil { + return + } + tls := ctx.NodePrometheusTLS + spec := &ds.Spec.Template.Spec + + spec.Volumes = append(spec.Volumes, tls.Volume()) + + for i := range spec.Containers { + c := &spec.Containers[i] + if c.Name != render.CalicoNodeObjectName { + continue + } + c.VolumeMounts = append(c.VolumeMounts, tls.VolumeMount(rmeta.OSTypeLinux)) + if tls.UseCertificateManagement() { + spec.InitContainers = append(spec.InitContainers, tls.InitContainer(common.CalicoNamespace, c.SecurityContext)) + } + } + + if ds.Spec.Template.Annotations == nil { + ds.Spec.Template.Annotations = map[string]string{} + } + ds.Spec.Template.Annotations[tls.HashAnnotationKey()] = tls.HashAnnotationValue() } // nodeEnterpriseEnv is the Enterprise felix configuration added to the diff --git a/pkg/enterprise/typha.go b/pkg/enterprise/typha.go index 5a2471a5fd..368e71e835 100644 --- a/pkg/enterprise/typha.go +++ b/pkg/enterprise/typha.go @@ -20,19 +20,16 @@ import ( rbacv1 "k8s.io/api/rbac/v1" "sigs.k8s.io/controller-runtime/pkg/client" + operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" ) func registerTypha() { - extensions.Modify(render.ComponentNameTypha, modifyTypha) + extensions.Modify(operatorv1.CalicoEnterprise, render.ComponentNameTypha, modifyTypha) } func modifyTypha(ctx extensions.RenderContext, objs []client.Object) []client.Object { - if ctx.Installation == nil || !ctx.Installation.Variant.IsEnterprise() { - return objs - } - if role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, "calico-typha"); ok { role.Rules = append(role.Rules, rbacv1.PolicyRule{ APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, diff --git a/pkg/extensions/factory.go b/pkg/extensions/factory.go index bf2d46f246..48d615fd6d 100644 --- a/pkg/extensions/factory.go +++ b/pkg/extensions/factory.go @@ -47,13 +47,14 @@ type Inputs struct { // // This is the generic seam controllers use to extend base operator behavior; // its first consumer is Calico Enterprise, but nothing here is enterprise -// specific. The core operator default does no side-effecting work. +// specific. A builder is registered per variant, so it only runs for its own +// variant and need not re-check it. type RenderContextBuilder func(in Inputs) (RenderContext, error) // BaseRenderContext maps the generically-gathered inputs onto a RenderContext. -// The default builder and every registered builder build on it, so the base -// fields are assembled in exactly one place. A builder layers its side-effect -// artifacts (e.g. NodePrometheusTLS) on top of the returned value. +// Every registered builder builds on it, so the base fields are assembled in +// exactly one place. A builder layers its side-effect artifacts (e.g. +// NodePrometheusTLS) on top of the returned value. func BaseRenderContext(in Inputs) RenderContext { return RenderContext{ Installation: in.Installation, @@ -63,20 +64,23 @@ func BaseRenderContext(in Inputs) RenderContext { } } -// defaultRenderContextBuilder is the core operator's builder. It does no -// side-effecting work and returns just the base RenderContext. -func defaultRenderContextBuilder(in Inputs) (RenderContext, error) { - return BaseRenderContext(in), nil -} +var renderContextBuilders = map[operatorv1.ProductVariant]RenderContextBuilder{} -var renderContextBuilder RenderContextBuilder = defaultRenderContextBuilder - -// RegisterRenderContextBuilder installs f as the builder the installation -// controller uses. Registration replaces any prior builder, so it is safe to -// call more than once. Without a registered builder the core operator default -// applies. -func RegisterRenderContextBuilder(f RenderContextBuilder) { renderContextBuilder = f } +// RegisterRenderContextBuilder installs f as the builder for the given variant. +// Registration replaces any prior builder, so it is safe to call more than +// once. Variants without a registered builder get the base render context. +func RegisterRenderContextBuilder(variant operatorv1.ProductVariant, f RenderContextBuilder) { + renderContextBuilders[variant] = f +} -// BuildRenderContext builds the RenderContext from in using the registered -// builder, or the core operator default when none is registered. -func BuildRenderContext(in Inputs) (RenderContext, error) { return renderContextBuilder(in) } +// BuildRenderContext builds the RenderContext from in using the builder +// registered for the installation variant, or the base render context when the +// variant has no registered builder. +func BuildRenderContext(in Inputs) (RenderContext, error) { + if in.Installation != nil { + if f, ok := renderContextBuilders[in.Installation.Variant]; ok { + return f(in) + } + } + return BaseRenderContext(in), nil +} diff --git a/pkg/extensions/factory_test.go b/pkg/extensions/factory_test.go index 7954f402bd..8b8ad825ce 100644 --- a/pkg/extensions/factory_test.go +++ b/pkg/extensions/factory_test.go @@ -27,7 +27,7 @@ import ( var _ = Describe("render context builder", func() { AfterEach(func() { extensions.ResetForTest() }) - It("returns the base render context from the default builder", func() { + It("returns the base render context when the variant has no builder", func() { install := &operatorv1.InstallationSpec{Variant: operatorv1.Calico} rc, err := extensions.BuildRenderContext(extensions.Inputs{ Installation: install, @@ -39,28 +39,44 @@ var _ = Describe("render context builder", func() { Expect(rc.NodePrometheusTLS).To(BeNil()) }) - It("uses a registered builder in place of the default", func() { - extensions.RegisterRenderContextBuilder(fakeBuilder(nil)) - rc, err := extensions.BuildRenderContext(extensions.Inputs{}) + It("uses the builder registered for the installation variant", func() { + extensions.RegisterRenderContextBuilder(operatorv1.CalicoEnterprise, fakeBuilder(nil)) + rc, err := extensions.BuildRenderContext(enterpriseInputs()) Expect(err).NotTo(HaveOccurred()) Expect(rc.ClusterDomain).To(Equal("from-fake")) }) + It("ignores a builder registered for a different variant", func() { + extensions.RegisterRenderContextBuilder(operatorv1.CalicoEnterprise, fakeBuilder(nil)) + rc, err := extensions.BuildRenderContext(extensions.Inputs{ + Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}, + ClusterDomain: "real", + }) + Expect(err).NotTo(HaveOccurred()) + Expect(rc.ClusterDomain).To(Equal("real")) + }) + It("surfaces the builder error", func() { - extensions.RegisterRenderContextBuilder(fakeBuilder(errors.New("boom"))) - _, err := extensions.BuildRenderContext(extensions.Inputs{}) + extensions.RegisterRenderContextBuilder(operatorv1.CalicoEnterprise, fakeBuilder(errors.New("boom"))) + _, err := extensions.BuildRenderContext(enterpriseInputs()) Expect(err).To(MatchError("boom")) }) - It("restores the default builder on reset", func() { - extensions.RegisterRenderContextBuilder(fakeBuilder(nil)) + It("restores the base builder on reset", func() { + extensions.RegisterRenderContextBuilder(operatorv1.CalicoEnterprise, fakeBuilder(nil)) extensions.ResetForTest() - rc, err := extensions.BuildRenderContext(extensions.Inputs{ClusterDomain: "real"}) + in := enterpriseInputs() + in.ClusterDomain = "real" + rc, err := extensions.BuildRenderContext(in) Expect(err).NotTo(HaveOccurred()) Expect(rc.ClusterDomain).To(Equal("real")) }) }) +func enterpriseInputs() extensions.Inputs { + return extensions.Inputs{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}} +} + func fakeBuilder(err error) extensions.RenderContextBuilder { return func(_ extensions.Inputs) (extensions.RenderContext, error) { if err != nil { diff --git a/pkg/extensions/image.go b/pkg/extensions/image.go index 776a02bffd..1063261993 100644 --- a/pkg/extensions/image.go +++ b/pkg/extensions/image.go @@ -20,15 +20,15 @@ import ( "github.com/tigera/operator/pkg/imageoverride" ) -// ImageOverride returns the component image to use for an installation, and -// false to decline (leaving the default in place). Implementations self-gate -// on in.Variant. +// ImageOverride returns the component image to use for an installation. An +// override is registered per variant, so it only runs for its own variant and +// need not re-check it. type ImageOverride = imageoverride.Override -// OverrideImage registers an image override under key. The key is the render -// component's image identifier (e.g. "node"). -func OverrideImage(key string, fn ImageOverride) { - imageoverride.Register(key, fn) +// OverrideImage registers an image override under key for the given variant. +// The key is the render component's image identifier (e.g. "node"). +func OverrideImage(variant operatorv1.ProductVariant, key string, fn ImageOverride) { + imageoverride.Register(variant, key, fn) } // ResolveImage returns the override registered for key if it applies to in, diff --git a/pkg/extensions/image_test.go b/pkg/extensions/image_test.go index 5a382d6c36..7f05c6cac1 100644 --- a/pkg/extensions/image_test.go +++ b/pkg/extensions/image_test.go @@ -28,19 +28,20 @@ var _ = Describe("image overrides", func() { extensions.ResetForTest() }) - It("uses the override when one matches", func() { - extensions.OverrideImage("node", func(in *operatorv1.InstallationSpec) (components.Component, bool) { - if !in.Variant.IsEnterprise() { - return components.Component{}, false - } - return components.ComponentTigeraNode, true + It("uses the override registered for the installation variant", func() { + extensions.OverrideImage(operatorv1.CalicoEnterprise, "node", func(in *operatorv1.InstallationSpec) components.Component { + return components.ComponentTigeraNode }) ent := &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise} Expect(extensions.ResolveImage("node", components.ComponentCalicoNode, ent)).To(Equal(components.ComponentTigeraNode)) }) - It("falls back to the default when no override matches", func() { + It("falls back to the default for a variant with no override", func() { + extensions.OverrideImage(operatorv1.CalicoEnterprise, "node", func(in *operatorv1.InstallationSpec) components.Component { + return components.ComponentTigeraNode + }) + calico := &operatorv1.InstallationSpec{Variant: operatorv1.Calico} Expect(extensions.ResolveImage("node", components.ComponentCalicoNode, calico)).To(Equal(components.ComponentCalicoNode)) }) diff --git a/pkg/extensions/modifier.go b/pkg/extensions/modifier.go index b872638531..7a6fbf8828 100644 --- a/pkg/extensions/modifier.go +++ b/pkg/extensions/modifier.go @@ -17,29 +17,40 @@ package extensions import ( "sigs.k8s.io/controller-runtime/pkg/client" + operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/imageoverride" ) // Modifier post-processes the objects a render component produced. It may mutate // matched objects and/or append additional objects, and must return the -// (possibly extended) slice. Implementations self-gate on ctx.Installation.Variant. +// (possibly extended) slice. A modifier is registered per variant, so it only +// runs for its own variant and need not re-check it. type Modifier func(ctx RenderContext, objs []client.Object) []client.Object -var modifiers = map[string]Modifier{} +type modifierKey struct { + variant operatorv1.ProductVariant + component string +} + +var modifiers = map[modifierKey]Modifier{} -// Modify registers fn as the modifier for the named component. A component has -// at most one modifier; the modifier handles all of that component's -// extension-specific mutations. Registration replaces any prior modifier, so it -// is idempotent and safe to call more than once - matching the image-override -// registry rather than stacking duplicate work. -func Modify(component string, fn Modifier) { - modifiers[component] = fn +// Modify registers fn as the modifier for the named component under the given +// variant. A (variant, component) pair has at most one modifier; it handles all +// of that component's extension-specific mutations for that variant. +// Registration replaces any prior modifier, so it is idempotent and safe to +// call more than once. +func Modify(variant operatorv1.ProductVariant, component string, fn Modifier) { + modifiers[modifierKey{variant, component}] = fn } -// ApplyModifiers runs the modifier registered for the named component over objs, -// returning objs unchanged when none is registered. +// ApplyModifiers runs the modifier registered for the named component and the +// installation's variant over objs, returning objs unchanged when none is +// registered (or when no installation is set). func ApplyModifiers(component string, ctx RenderContext, objs []client.Object) []client.Object { - if fn, ok := modifiers[component]; ok { + if ctx.Installation == nil { + return objs + } + if fn, ok := modifiers[modifierKey{ctx.Installation.Variant, component}]; ok { objs = fn(ctx, objs) } return objs @@ -59,7 +70,7 @@ func FindObject[T client.Object](objs []client.Object, name string) (T, bool) { // ResetForTest clears every registry: modifiers, image overrides, and the // render context builder. Test-only. func ResetForTest() { - modifiers = map[string]Modifier{} - renderContextBuilder = defaultRenderContextBuilder + modifiers = map[modifierKey]Modifier{} + renderContextBuilders = map[operatorv1.ProductVariant]RenderContextBuilder{} imageoverride.ResetForTest() } diff --git a/pkg/extensions/modifier_test.go b/pkg/extensions/modifier_test.go index 7b40beb740..82decda600 100644 --- a/pkg/extensions/modifier_test.go +++ b/pkg/extensions/modifier_test.go @@ -21,6 +21,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" + operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/extensions" ) @@ -29,8 +30,10 @@ var _ = Describe("modifier registry", func() { extensions.ResetForTest() }) - It("applies a registered modifier to the matching component", func() { - extensions.Modify("test", func(ctx extensions.RenderContext, objs []client.Object) []client.Object { + entCtx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}} + + It("applies a registered modifier to the matching component and variant", func() { + extensions.Modify(operatorv1.CalicoEnterprise, "test", func(ctx extensions.RenderContext, objs []client.Object) []client.Object { cm, ok := extensions.FindObject[*corev1.ConfigMap](objs, "cm") Expect(ok).To(BeTrue()) cm.Data = map[string]string{"k": "v"} @@ -38,7 +41,7 @@ var _ = Describe("modifier registry", func() { }) in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} - out := extensions.ApplyModifiers("test", extensions.RenderContext{}, in) + out := extensions.ApplyModifiers("test", entCtx, in) Expect(out).To(HaveLen(2)) cm := out[0].(*corev1.ConfigMap) @@ -48,20 +51,35 @@ var _ = Describe("modifier registry", func() { It("returns objects unchanged when no modifier is registered", func() { in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} - out := extensions.ApplyModifiers("unregistered", extensions.RenderContext{}, in) + out := extensions.ApplyModifiers("unregistered", entCtx, in) Expect(out).To(Equal(in)) }) - It("replaces rather than stacks when a component is registered twice", func() { + It("does not apply a modifier registered for a different variant", func() { + extensions.Modify(operatorv1.CalicoEnterprise, "test", func(_ extensions.RenderContext, objs []client.Object) []client.Object { + return append(objs, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "extra"}}) + }) + + calicoCtx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}} + in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} + Expect(extensions.ApplyModifiers("test", calicoCtx, in)).To(Equal(in)) + }) + + It("returns objects unchanged when no installation is set", func() { + in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} + Expect(extensions.ApplyModifiers("test", extensions.RenderContext{}, in)).To(Equal(in)) + }) + + It("replaces rather than stacks when a (variant, component) is registered twice", func() { add := func(name string) extensions.Modifier { return func(_ extensions.RenderContext, objs []client.Object) []client.Object { return append(objs, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: name}}) } } - extensions.Modify("test", add("first")) - extensions.Modify("test", add("second")) + extensions.Modify(operatorv1.CalicoEnterprise, "test", add("first")) + extensions.Modify(operatorv1.CalicoEnterprise, "test", add("second")) - out := extensions.ApplyModifiers("test", extensions.RenderContext{}, nil) + out := extensions.ApplyModifiers("test", entCtx, nil) Expect(out).To(HaveLen(1)) Expect(out[0].GetName()).To(Equal("second")) }) diff --git a/pkg/extensions/rendercontext.go b/pkg/extensions/rendercontext.go index 52d0d753c5..9ea25e1b6b 100644 --- a/pkg/extensions/rendercontext.go +++ b/pkg/extensions/rendercontext.go @@ -36,10 +36,9 @@ type RenderContext struct { // TrustedBundle is the shared CA bundle for the calico-system namespace. TrustedBundle certificatemanagement.TrustedBundle - // NodePrometheusTLS is created by the enterprise RenderContextFactory (it has - // cluster side effects, so it can't be built in a modifier). Two consumers - // read it: the installation controller passes it to the node configuration - // (PrometheusServerTLS) so the keypair is mounted, and the node modifier uses - // it to set the FELIX_PROMETHEUSREPORTER* certificate env vars. + // NodePrometheusTLS is created by the enterprise render context builder (it + // has cluster side effects, so it can't be built in a modifier). The node + // modifier is its only consumer: it mounts the keypair onto the daemonset and + // sets the FELIX_PROMETHEUSREPORTER* certificate env vars. NodePrometheusTLS certificatemanagement.KeyPairInterface } diff --git a/pkg/imageoverride/imageoverride.go b/pkg/imageoverride/imageoverride.go index bf8950cf84..c3bc471bc2 100644 --- a/pkg/imageoverride/imageoverride.go +++ b/pkg/imageoverride/imageoverride.go @@ -22,30 +22,35 @@ import ( "github.com/tigera/operator/pkg/components" ) -// Override selects the component image to use for an installation, returning -// false to decline (leaving the default in place). -type Override func(in *operatorv1.InstallationSpec) (components.Component, bool) +// Override selects the component image to use for an installation. +type Override func(in *operatorv1.InstallationSpec) components.Component -var registry = map[string]Override{} +type overrideKey struct { + variant operatorv1.ProductVariant + key string +} + +var registry = map[overrideKey]Override{} -// Register stores fn under key. The key is the render component's image -// identifier (e.g. "node"). -func Register(key string, fn Override) { - registry[key] = fn +// Register stores fn under key for the given variant. The key is the render +// component's image identifier (e.g. "node"). +func Register(variant operatorv1.ProductVariant, key string, fn Override) { + registry[overrideKey{variant, key}] = fn } -// Resolve returns the override registered for key if it applies to in, -// otherwise def. +// Resolve returns the override registered for key under the installation's +// variant, otherwise def. func Resolve(key string, def components.Component, in *operatorv1.InstallationSpec) components.Component { - if fn, ok := registry[key]; ok { - if c, override := fn(in); override { - return c - } + if in == nil { + return def + } + if fn, ok := registry[overrideKey{in.Variant, key}]; ok { + return fn(in) } return def } // ResetForTest clears the registry. Test-only. func ResetForTest() { - registry = map[string]Override{} + registry = map[overrideKey]Override{} } diff --git a/pkg/render/component.go b/pkg/render/component.go index 0fa4f5da8b..7785834dcf 100644 --- a/pkg/render/component.go +++ b/pkg/render/component.go @@ -41,18 +41,16 @@ type Component interface { } // Extensible is implemented by components that expose extension points. The -// componentHandler uses Name() to look up registered modifiers. Components -// without extensions need not implement it. -// -// Note this interface is structural: any component that grows a Name() string -// method for an unrelated reason becomes modifier-eligible. There are no name -// collisions today, but keep it in mind when adding a Name() method. +// componentHandler uses ModifierKey() to look up registered modifiers. +// Components without extensions need not implement it. The method name is +// deliberately specific (not a generic Name()) so an unrelated method can't +// make a component modifier-eligible by accident. type Extensible interface { - Name() string + ModifierKey() string } // Component names used as keys into the extension modifier registry. Keep these -// in sync with the Name() methods that return them. +// in sync with the ModifierKey() methods that return them. const ( ComponentNameTypha = "typha" ComponentNameNode = "node" diff --git a/pkg/render/node.go b/pkg/render/node.go index 0e42fe3091..a8d397e1a1 100644 --- a/pkg/render/node.go +++ b/pkg/render/node.go @@ -127,8 +127,6 @@ type NodeConfiguration struct { // For details on why this is needed see 'Node and Installation finalizer' in the core_controller. CanRemoveCNIFinalizer bool - PrometheusServerTLS certificatemanagement.KeyPairInterface - // BGPLayouts is returned by the rendering code after modifying its namespace // so that it can be deployed into the cluster. // TODO: The controller should pass the contents, the renderer should build its own @@ -193,7 +191,7 @@ func (c *nodeComponent) SupportedOSType() rmeta.OSType { return rmeta.OSTypeLinux } -func (c *nodeComponent) Name() string { +func (c *nodeComponent) ModifierKey() string { return ComponentNameNode } @@ -897,18 +895,11 @@ func (c *nodeComponent) nodeDaemonset(cniCfgMap *corev1.ConfigMap) *appsv1.Daemo if len(c.cfg.BirdTemplates) != 0 { annotations[birdTemplateHashAnnotation] = rmeta.AnnotationHash(c.cfg.BirdTemplates) } - if c.cfg.PrometheusServerTLS != nil { - annotations[c.cfg.PrometheusServerTLS.HashAnnotationKey()] = c.cfg.PrometheusServerTLS.HashAnnotationValue() - } if c.cfg.TLS.NodeSecret.UseCertificateManagement() { initContainers = append(initContainers, c.cfg.TLS.NodeSecret.InitContainer(common.CalicoNamespace, nodeContainer.SecurityContext)) } - if c.cfg.PrometheusServerTLS != nil && c.cfg.PrometheusServerTLS.UseCertificateManagement() { - initContainers = append(initContainers, c.cfg.PrometheusServerTLS.InitContainer(common.CalicoNamespace, nodeContainer.SecurityContext)) - } - if cniCfgMap != nil { annotations[nodeCniConfigAnnotation] = rmeta.AnnotationHash(cniCfgMap.Data) } @@ -1114,10 +1105,6 @@ func (c *nodeComponent) nodeVolumes() []corev1.Volume { }, }) } - if c.cfg.PrometheusServerTLS != nil { - volumes = append(volumes, c.cfg.PrometheusServerTLS.Volume()) - } - return volumes } @@ -1358,9 +1345,6 @@ func (c *nodeComponent) nodeVolumeMounts() []corev1.VolumeMount { SubPath: BGPLayoutConfigMapKey, }) } - if c.cfg.PrometheusServerTLS != nil { - nodeVolumeMounts = append(nodeVolumeMounts, c.cfg.PrometheusServerTLS.VolumeMount(c.SupportedOSType())) - } return nodeVolumeMounts } diff --git a/pkg/render/node_enterprise_test.go b/pkg/render/node_enterprise_test.go index 4cc73d2a70..9e0d9c493b 100644 --- a/pkg/render/node_enterprise_test.go +++ b/pkg/render/node_enterprise_test.go @@ -33,6 +33,7 @@ import ( "github.com/tigera/operator/pkg/dns" "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" + rmeta "github.com/tigera/operator/pkg/render/common/meta" ) // These tests run the real node/typha render output through the registered @@ -99,13 +100,12 @@ var _ = Describe("node enterprise modifier integration", func() { // modifier, exactly as the componentHandler does. renderNodeObjects := func() []client.Object { cfg := &render.NodeConfiguration{ - K8sServiceEp: k8sapi.ServiceEndpoint{}, - Installation: instance, - TLS: typhaNodeTLS, - ClusterDomain: dns.DefaultClusterDomain, - FelixHealthPort: 9099, - IPPools: instance.CalicoNetwork.IPPools, - PrometheusServerTLS: renderCtx.NodePrometheusTLS, + K8sServiceEp: k8sapi.ServiceEndpoint{}, + Installation: instance, + TLS: typhaNodeTLS, + ClusterDomain: dns.DefaultClusterDomain, + FelixHealthPort: 9099, + IPPools: instance.CalicoNetwork.IPPools, } comp := render.Node(cfg) Expect(comp.ResolveImages(nil)).NotTo(HaveOccurred()) @@ -145,8 +145,11 @@ var _ = Describe("node enterprise modifier integration", func() { corev1.EnvVar{Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, )) // The reporter cert env is wired from the NodePrometheusTLS keypair the - // factory creates. + // builder creates, and the modifier mounts that keypair onto the daemonset. Expect(c.Env).To(ContainElement(HaveField("Name", "FELIX_PROMETHEUSREPORTERCERTFILE"))) + Expect(ds.Spec.Template.Spec.Volumes).To(ContainElement(renderCtx.NodePrometheusTLS.Volume())) + Expect(c.VolumeMounts).To(ContainElement(renderCtx.NodePrometheusTLS.VolumeMount(rmeta.OSTypeLinux))) + Expect(ds.Spec.Template.Annotations).To(HaveKey(renderCtx.NodePrometheusTLS.HashAnnotationKey())) // BGP is enabled, so the bird readiness check is present and the modifier // adds the BGP metrics check. diff --git a/pkg/render/typha.go b/pkg/render/typha.go index 9b1209a3e7..3632978f81 100644 --- a/pkg/render/typha.go +++ b/pkg/render/typha.go @@ -113,7 +113,7 @@ func (c *typhaComponent) SupportedOSType() rmeta.OSType { return rmeta.OSTypeLinux } -func (c *typhaComponent) Name() string { return ComponentNameTypha } +func (c *typhaComponent) ModifierKey() string { return ComponentNameTypha } func (c *typhaComponent) Objects() ([]client.Object, []client.Object) { pdb := c.typhaPodDisruptionBudget() From 6a33636c85b4aee3a0e91d51f963dc0c66afec74 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Tue, 9 Jun 2026 11:14:16 -0700 Subject: [PATCH 18/38] Collapse extension seams into Extension and Setup Merge the per-component image override and modifier into a single Extension{Image, Modify} registered once per (variant, component) via extensions.Register, so all of a component's variance lives in one place. The image half still lands in the imageoverride leaf so render resolves it without an import cycle; the fan-out is internal to Register. Rename the variant-level render context builder to Setup (RegisterSetup/RunSetup). That names the two phases a reader has to hold: Setup is the controller-side work that builds the RenderContext baton, and Extension hooks are the pure render-time funcs. Three registries with three key schemes become two concepts split by when they run. --- .../installation/core_controller.go | 13 ++--- pkg/controller/utils/component_test.go | 10 ++-- pkg/enterprise/installation.go | 11 ++-- pkg/enterprise/installation_test.go | 8 +-- pkg/enterprise/node.go | 8 +-- pkg/enterprise/typha.go | 4 +- pkg/extensions/{modifier.go => extension.go} | 43 ++++++++++----- .../{modifier_test.go => extension_test.go} | 32 ++++++----- pkg/extensions/image.go | 15 ++---- pkg/extensions/image_test.go | 12 +++-- pkg/extensions/{factory.go => setup.go} | 53 +++++++++---------- .../{factory_test.go => setup_test.go} | 32 +++++------ 12 files changed, 135 insertions(+), 106 deletions(-) rename pkg/extensions/{modifier.go => extension.go} (58%) rename pkg/extensions/{modifier_test.go => extension_test.go} (71%) rename pkg/extensions/{factory.go => setup.go} (50%) rename pkg/extensions/{factory_test.go => setup_test.go} (65%) diff --git a/pkg/controller/installation/core_controller.go b/pkg/controller/installation/core_controller.go index 568a59f954..c173827d23 100644 --- a/pkg/controller/installation/core_controller.go +++ b/pkg/controller/installation/core_controller.go @@ -1213,12 +1213,13 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile calicoVersion = components.EnterpriseRelease } - // Build the render context handed to registered modifiers. The core operator - // builder returns just the base context; an extension's builder additionally - // does controller-side work for its variant - validating config and creating - // the node-prometheus certificate, adding it (and the prometheus/esgw certs) - // to the trusted bundle - and may abort the reconcile by returning an error. - modCtx, err := extensions.BuildRenderContext(extensions.Inputs{ + // Run the variant setup to build the render context handed to registered + // modifiers. The core operator has no setup, so it gets just the base + // context; an extension's setup additionally does controller-side work for + // its variant - validating config and creating the node-prometheus + // certificate, adding it (and the prometheus/esgw certs) to the trusted + // bundle - and may abort the reconcile by returning an error. + modCtx, err := extensions.RunSetup(extensions.Inputs{ Ctx: ctx, Client: r.client, Installation: &instance.Spec, diff --git a/pkg/controller/utils/component_test.go b/pkg/controller/utils/component_test.go index eeb989e501..93c888812d 100644 --- a/pkg/controller/utils/component_test.go +++ b/pkg/controller/utils/component_test.go @@ -2493,10 +2493,12 @@ var _ = Describe("componentHandler modifier application", func() { }) It("applies registered modifiers to a named component before create", func() { - extensions.Modify(operatorv1.CalicoEnterprise, "fake", func(ctx extensions.RenderContext, objs []client.Object) []client.Object { - cm := objs[0].(*corev1.ConfigMap) - cm.Data = map[string]string{"patched": "yes"} - return objs + extensions.Register(operatorv1.CalicoEnterprise, "fake", extensions.Extension{ + Modify: func(ctx extensions.RenderContext, objs []client.Object) []client.Object { + cm := objs[0].(*corev1.ConfigMap) + cm.Data = map[string]string{"patched": "yes"} + return objs + }, }) s := runtime.NewScheme() diff --git a/pkg/enterprise/installation.go b/pkg/enterprise/installation.go index cd49480c68..ffb0dd7c98 100644 --- a/pkg/enterprise/installation.go +++ b/pkg/enterprise/installation.go @@ -28,14 +28,13 @@ import ( ) func registerInstallation() { - extensions.RegisterRenderContextBuilder(operatorv1.CalicoEnterprise, buildRenderContext) + extensions.RegisterSetup(operatorv1.CalicoEnterprise, setup) } -// buildRenderContext is the Calico Enterprise RenderContextBuilder. It builds -// the base render context and then does the controller-side work the modifiers -// can't: validating config and creating/fetching the certificates that feed the -// trusted bundle. -func buildRenderContext(in extensions.Inputs) (extensions.RenderContext, error) { +// setup is the Calico Enterprise setup phase. It builds the base render context +// and then does the controller-side work the modifiers can't: validating config +// and creating/fetching the certificates that feed the trusted bundle. +func setup(in extensions.Inputs) (extensions.RenderContext, error) { rc := extensions.BaseRenderContext(in) // Reject the unsupported zero reporter port. The port value itself is derived diff --git a/pkg/enterprise/installation_test.go b/pkg/enterprise/installation_test.go index f3aa09f887..1ebf7bf52b 100644 --- a/pkg/enterprise/installation_test.go +++ b/pkg/enterprise/installation_test.go @@ -32,7 +32,7 @@ import ( "github.com/tigera/operator/pkg/extensions" ) -var _ = Describe("installation render context builder", func() { +var _ = Describe("installation setup", func() { BeforeEach(func() { enterprise.Register() }) AfterEach(func() { extensions.ResetForTest() }) @@ -42,18 +42,18 @@ var _ = Describe("installation render context builder", func() { in.FelixConfiguration = &v3.FelixConfiguration{ Spec: v3.FelixConfigurationSpec{PrometheusReporterPort: &port}, } - _, err := extensions.BuildRenderContext(in) + _, err := extensions.RunSetup(in) Expect(err).To(HaveOccurred()) }) It("creates the node prometheus keypair for the enterprise variant", func() { - rc, err := extensions.BuildRenderContext(newInputs(operatorv1.CalicoEnterprise)) + rc, err := extensions.RunSetup(newInputs(operatorv1.CalicoEnterprise)) Expect(err).NotTo(HaveOccurred()) Expect(rc.NodePrometheusTLS).NotTo(BeNil()) }) It("is a no-op for the Calico variant", func() { - rc, err := extensions.BuildRenderContext(newInputs(operatorv1.Calico)) + rc, err := extensions.RunSetup(newInputs(operatorv1.Calico)) Expect(err).NotTo(HaveOccurred()) Expect(rc.NodePrometheusTLS).To(BeNil()) }) diff --git a/pkg/enterprise/node.go b/pkg/enterprise/node.go index 455c7b403f..335bac63c1 100644 --- a/pkg/enterprise/node.go +++ b/pkg/enterprise/node.go @@ -47,10 +47,12 @@ const ( ) func registerNode() { - extensions.OverrideImage(operatorv1.CalicoEnterprise, render.ComponentNameNode, func(in *operatorv1.InstallationSpec) components.Component { - return components.ComponentTigeraNode + extensions.Register(operatorv1.CalicoEnterprise, render.ComponentNameNode, extensions.Extension{ + Image: func(in *operatorv1.InstallationSpec) components.Component { + return components.ComponentTigeraNode + }, + Modify: modifyNode, }) - extensions.Modify(operatorv1.CalicoEnterprise, render.ComponentNameNode, modifyNode) } // modifyNode layers Calico Enterprise behavior onto the rendered calico/node diff --git a/pkg/enterprise/typha.go b/pkg/enterprise/typha.go index 368e71e835..457243842a 100644 --- a/pkg/enterprise/typha.go +++ b/pkg/enterprise/typha.go @@ -26,7 +26,9 @@ import ( ) func registerTypha() { - extensions.Modify(operatorv1.CalicoEnterprise, render.ComponentNameTypha, modifyTypha) + extensions.Register(operatorv1.CalicoEnterprise, render.ComponentNameTypha, extensions.Extension{ + Modify: modifyTypha, + }) } func modifyTypha(ctx extensions.RenderContext, objs []client.Object) []client.Object { diff --git a/pkg/extensions/modifier.go b/pkg/extensions/extension.go similarity index 58% rename from pkg/extensions/modifier.go rename to pkg/extensions/extension.go index 7a6fbf8828..4f57d64dd2 100644 --- a/pkg/extensions/modifier.go +++ b/pkg/extensions/extension.go @@ -21,10 +21,24 @@ import ( "github.com/tigera/operator/pkg/imageoverride" ) +// Extension is everything a variant layers onto one render component. Every +// field is optional: a component that only needs a different image sets Image +// and leaves Modify nil, and vice versa. This is the single registration a +// variant makes per component, so all of that component's variance lives in one +// place. +type Extension struct { + // Image overrides the component's image. Resolved during ResolveImages, in + // the render package, via the imageoverride leaf. + Image ImageOverride + + // Modify post-processes the component's rendered objects, after Objects(). + Modify Modifier +} + // Modifier post-processes the objects a render component produced. It may mutate // matched objects and/or append additional objects, and must return the -// (possibly extended) slice. A modifier is registered per variant, so it only -// runs for its own variant and need not re-check it. +// (possibly extended) slice. A modifier runs only for the variant it was +// registered under, so it need not re-check the variant. type Modifier func(ctx RenderContext, objs []client.Object) []client.Object type modifierKey struct { @@ -34,13 +48,18 @@ type modifierKey struct { var modifiers = map[modifierKey]Modifier{} -// Modify registers fn as the modifier for the named component under the given -// variant. A (variant, component) pair has at most one modifier; it handles all -// of that component's extension-specific mutations for that variant. -// Registration replaces any prior modifier, so it is idempotent and safe to -// call more than once. -func Modify(variant operatorv1.ProductVariant, component string, fn Modifier) { - modifiers[modifierKey{variant, component}] = fn +// Register installs e as the extension for the named component under the given +// variant. A (variant, component) pair has at most one extension; registration +// replaces any prior one, so it is idempotent and safe to call more than once. +// The image override lives in the imageoverride leaf (so the render package can +// resolve it without an import cycle); the modifier lives here. +func Register(variant operatorv1.ProductVariant, component string, e Extension) { + if e.Image != nil { + imageoverride.Register(variant, component, e.Image) + } + if e.Modify != nil { + modifiers[modifierKey{variant, component}] = e.Modify + } } // ApplyModifiers runs the modifier registered for the named component and the @@ -67,10 +86,10 @@ func FindObject[T client.Object](objs []client.Object, name string) (T, bool) { return zero, false } -// ResetForTest clears every registry: modifiers, image overrides, and the -// render context builder. Test-only. +// ResetForTest clears every registry: modifiers, image overrides, and variant +// setups. Test-only. func ResetForTest() { modifiers = map[modifierKey]Modifier{} - renderContextBuilders = map[operatorv1.ProductVariant]RenderContextBuilder{} + setups = map[operatorv1.ProductVariant]Setup{} imageoverride.ResetForTest() } diff --git a/pkg/extensions/modifier_test.go b/pkg/extensions/extension_test.go similarity index 71% rename from pkg/extensions/modifier_test.go rename to pkg/extensions/extension_test.go index 82decda600..c935dac9f0 100644 --- a/pkg/extensions/modifier_test.go +++ b/pkg/extensions/extension_test.go @@ -25,7 +25,7 @@ import ( "github.com/tigera/operator/pkg/extensions" ) -var _ = Describe("modifier registry", func() { +var _ = Describe("extension registry", func() { AfterEach(func() { extensions.ResetForTest() }) @@ -33,11 +33,13 @@ var _ = Describe("modifier registry", func() { entCtx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}} It("applies a registered modifier to the matching component and variant", func() { - extensions.Modify(operatorv1.CalicoEnterprise, "test", func(ctx extensions.RenderContext, objs []client.Object) []client.Object { - cm, ok := extensions.FindObject[*corev1.ConfigMap](objs, "cm") - Expect(ok).To(BeTrue()) - cm.Data = map[string]string{"k": "v"} - return append(objs, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "extra"}}) + extensions.Register(operatorv1.CalicoEnterprise, "test", extensions.Extension{ + Modify: func(ctx extensions.RenderContext, objs []client.Object) []client.Object { + cm, ok := extensions.FindObject[*corev1.ConfigMap](objs, "cm") + Expect(ok).To(BeTrue()) + cm.Data = map[string]string{"k": "v"} + return append(objs, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "extra"}}) + }, }) in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} @@ -56,8 +58,10 @@ var _ = Describe("modifier registry", func() { }) It("does not apply a modifier registered for a different variant", func() { - extensions.Modify(operatorv1.CalicoEnterprise, "test", func(_ extensions.RenderContext, objs []client.Object) []client.Object { - return append(objs, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "extra"}}) + extensions.Register(operatorv1.CalicoEnterprise, "test", extensions.Extension{ + Modify: func(_ extensions.RenderContext, objs []client.Object) []client.Object { + return append(objs, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "extra"}}) + }, }) calicoCtx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}} @@ -71,13 +75,15 @@ var _ = Describe("modifier registry", func() { }) It("replaces rather than stacks when a (variant, component) is registered twice", func() { - add := func(name string) extensions.Modifier { - return func(_ extensions.RenderContext, objs []client.Object) []client.Object { - return append(objs, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: name}}) + add := func(name string) extensions.Extension { + return extensions.Extension{ + Modify: func(_ extensions.RenderContext, objs []client.Object) []client.Object { + return append(objs, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: name}}) + }, } } - extensions.Modify(operatorv1.CalicoEnterprise, "test", add("first")) - extensions.Modify(operatorv1.CalicoEnterprise, "test", add("second")) + extensions.Register(operatorv1.CalicoEnterprise, "test", add("first")) + extensions.Register(operatorv1.CalicoEnterprise, "test", add("second")) out := extensions.ApplyModifiers("test", entCtx, nil) Expect(out).To(HaveLen(1)) diff --git a/pkg/extensions/image.go b/pkg/extensions/image.go index 1063261993..477c89deef 100644 --- a/pkg/extensions/image.go +++ b/pkg/extensions/image.go @@ -20,19 +20,14 @@ import ( "github.com/tigera/operator/pkg/imageoverride" ) -// ImageOverride returns the component image to use for an installation. An -// override is registered per variant, so it only runs for its own variant and -// need not re-check it. +// ImageOverride returns the component image to use for an installation. It is +// the Image field of an Extension. An override runs only for the variant it was +// registered under, so it need not re-check the variant. type ImageOverride = imageoverride.Override -// OverrideImage registers an image override under key for the given variant. -// The key is the render component's image identifier (e.g. "node"). -func OverrideImage(variant operatorv1.ProductVariant, key string, fn ImageOverride) { - imageoverride.Register(variant, key, fn) -} - // ResolveImage returns the override registered for key if it applies to in, -// otherwise def. Render components call this inside ResolveImages. +// otherwise def. The render package resolves images through the imageoverride +// leaf directly; this is the same lookup for callers already inside extensions. func ResolveImage(key string, def components.Component, in *operatorv1.InstallationSpec) components.Component { return imageoverride.Resolve(key, def, in) } diff --git a/pkg/extensions/image_test.go b/pkg/extensions/image_test.go index 7f05c6cac1..25926c1a4f 100644 --- a/pkg/extensions/image_test.go +++ b/pkg/extensions/image_test.go @@ -29,8 +29,10 @@ var _ = Describe("image overrides", func() { }) It("uses the override registered for the installation variant", func() { - extensions.OverrideImage(operatorv1.CalicoEnterprise, "node", func(in *operatorv1.InstallationSpec) components.Component { - return components.ComponentTigeraNode + extensions.Register(operatorv1.CalicoEnterprise, "node", extensions.Extension{ + Image: func(in *operatorv1.InstallationSpec) components.Component { + return components.ComponentTigeraNode + }, }) ent := &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise} @@ -38,8 +40,10 @@ var _ = Describe("image overrides", func() { }) It("falls back to the default for a variant with no override", func() { - extensions.OverrideImage(operatorv1.CalicoEnterprise, "node", func(in *operatorv1.InstallationSpec) components.Component { - return components.ComponentTigeraNode + extensions.Register(operatorv1.CalicoEnterprise, "node", extensions.Extension{ + Image: func(in *operatorv1.InstallationSpec) components.Component { + return components.ComponentTigeraNode + }, }) calico := &operatorv1.InstallationSpec{Variant: operatorv1.Calico} diff --git a/pkg/extensions/factory.go b/pkg/extensions/setup.go similarity index 50% rename from pkg/extensions/factory.go rename to pkg/extensions/setup.go index 48d615fd6d..136863b0ff 100644 --- a/pkg/extensions/factory.go +++ b/pkg/extensions/setup.go @@ -25,11 +25,11 @@ import ( "github.com/tigera/operator/pkg/tls/certificatemanagement" ) -// Inputs is the reconcile state a RenderContextBuilder builds a RenderContext -// from. The installation controller populates it directly. It carries both the -// values that flow straight into the RenderContext and the side-effecting -// dependencies (Client, CertificateManager) a builder needs to produce -// controller-side artifacts. +// Inputs is the reconcile state a Setup builds a RenderContext from. The +// installation controller populates it directly. It carries both the values +// that flow straight into the RenderContext and the side-effecting dependencies +// (Client, CertificateManager) a setup needs to produce controller-side +// artifacts. type Inputs struct { Ctx context.Context Client client.Client @@ -40,21 +40,21 @@ type Inputs struct { ClusterDomain string } -// RenderContextBuilder builds the RenderContext handed to render modifiers. It -// performs any controller-side work (creating certificates, extending the -// trusted bundle) and returns the assembled RenderContext - or an error that -// aborts the reconcile. +// Setup is a variant's controller-side reconcile phase. It performs the work +// modifiers can't (creating certificates, extending the trusted bundle, +// validating config) and returns the RenderContext that is then handed to that +// variant's modifiers - or an error that aborts the reconcile. // // This is the generic seam controllers use to extend base operator behavior; // its first consumer is Calico Enterprise, but nothing here is enterprise -// specific. A builder is registered per variant, so it only runs for its own -// variant and need not re-check it. -type RenderContextBuilder func(in Inputs) (RenderContext, error) +// specific. A setup runs only for the variant it was registered under, so it +// need not re-check the variant. +type Setup func(in Inputs) (RenderContext, error) // BaseRenderContext maps the generically-gathered inputs onto a RenderContext. -// Every registered builder builds on it, so the base fields are assembled in -// exactly one place. A builder layers its side-effect artifacts (e.g. -// NodePrometheusTLS) on top of the returned value. +// Every setup builds on it, so the base fields are assembled in exactly one +// place. A setup layers its side-effect artifacts (e.g. NodePrometheusTLS) on +// top of the returned value. func BaseRenderContext(in Inputs) RenderContext { return RenderContext{ Installation: in.Installation, @@ -64,22 +64,21 @@ func BaseRenderContext(in Inputs) RenderContext { } } -var renderContextBuilders = map[operatorv1.ProductVariant]RenderContextBuilder{} +var setups = map[operatorv1.ProductVariant]Setup{} -// RegisterRenderContextBuilder installs f as the builder for the given variant. -// Registration replaces any prior builder, so it is safe to call more than -// once. Variants without a registered builder get the base render context. -func RegisterRenderContextBuilder(variant operatorv1.ProductVariant, f RenderContextBuilder) { - renderContextBuilders[variant] = f +// RegisterSetup installs s as the setup for the given variant. Registration +// replaces any prior setup, so it is safe to call more than once. Variants +// without a registered setup get the base render context. +func RegisterSetup(variant operatorv1.ProductVariant, s Setup) { + setups[variant] = s } -// BuildRenderContext builds the RenderContext from in using the builder -// registered for the installation variant, or the base render context when the -// variant has no registered builder. -func BuildRenderContext(in Inputs) (RenderContext, error) { +// RunSetup runs the setup registered for the installation variant and returns +// its RenderContext, or the base render context when the variant has no setup. +func RunSetup(in Inputs) (RenderContext, error) { if in.Installation != nil { - if f, ok := renderContextBuilders[in.Installation.Variant]; ok { - return f(in) + if s, ok := setups[in.Installation.Variant]; ok { + return s(in) } } return BaseRenderContext(in), nil diff --git a/pkg/extensions/factory_test.go b/pkg/extensions/setup_test.go similarity index 65% rename from pkg/extensions/factory_test.go rename to pkg/extensions/setup_test.go index 8b8ad825ce..9967bfe209 100644 --- a/pkg/extensions/factory_test.go +++ b/pkg/extensions/setup_test.go @@ -24,12 +24,12 @@ import ( "github.com/tigera/operator/pkg/extensions" ) -var _ = Describe("render context builder", func() { +var _ = Describe("variant setup", func() { AfterEach(func() { extensions.ResetForTest() }) - It("returns the base render context when the variant has no builder", func() { + It("returns the base render context when the variant has no setup", func() { install := &operatorv1.InstallationSpec{Variant: operatorv1.Calico} - rc, err := extensions.BuildRenderContext(extensions.Inputs{ + rc, err := extensions.RunSetup(extensions.Inputs{ Installation: install, ClusterDomain: "cluster.local", }) @@ -39,16 +39,16 @@ var _ = Describe("render context builder", func() { Expect(rc.NodePrometheusTLS).To(BeNil()) }) - It("uses the builder registered for the installation variant", func() { - extensions.RegisterRenderContextBuilder(operatorv1.CalicoEnterprise, fakeBuilder(nil)) - rc, err := extensions.BuildRenderContext(enterpriseInputs()) + It("uses the setup registered for the installation variant", func() { + extensions.RegisterSetup(operatorv1.CalicoEnterprise, fakeSetup(nil)) + rc, err := extensions.RunSetup(enterpriseInputs()) Expect(err).NotTo(HaveOccurred()) Expect(rc.ClusterDomain).To(Equal("from-fake")) }) - It("ignores a builder registered for a different variant", func() { - extensions.RegisterRenderContextBuilder(operatorv1.CalicoEnterprise, fakeBuilder(nil)) - rc, err := extensions.BuildRenderContext(extensions.Inputs{ + It("ignores a setup registered for a different variant", func() { + extensions.RegisterSetup(operatorv1.CalicoEnterprise, fakeSetup(nil)) + rc, err := extensions.RunSetup(extensions.Inputs{ Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}, ClusterDomain: "real", }) @@ -56,18 +56,18 @@ var _ = Describe("render context builder", func() { Expect(rc.ClusterDomain).To(Equal("real")) }) - It("surfaces the builder error", func() { - extensions.RegisterRenderContextBuilder(operatorv1.CalicoEnterprise, fakeBuilder(errors.New("boom"))) - _, err := extensions.BuildRenderContext(enterpriseInputs()) + It("surfaces the setup error", func() { + extensions.RegisterSetup(operatorv1.CalicoEnterprise, fakeSetup(errors.New("boom"))) + _, err := extensions.RunSetup(enterpriseInputs()) Expect(err).To(MatchError("boom")) }) - It("restores the base builder on reset", func() { - extensions.RegisterRenderContextBuilder(operatorv1.CalicoEnterprise, fakeBuilder(nil)) + It("restores the base context on reset", func() { + extensions.RegisterSetup(operatorv1.CalicoEnterprise, fakeSetup(nil)) extensions.ResetForTest() in := enterpriseInputs() in.ClusterDomain = "real" - rc, err := extensions.BuildRenderContext(in) + rc, err := extensions.RunSetup(in) Expect(err).NotTo(HaveOccurred()) Expect(rc.ClusterDomain).To(Equal("real")) }) @@ -77,7 +77,7 @@ func enterpriseInputs() extensions.Inputs { return extensions.Inputs{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}} } -func fakeBuilder(err error) extensions.RenderContextBuilder { +func fakeSetup(err error) extensions.Setup { return func(_ extensions.Inputs) (extensions.RenderContext, error) { if err != nil { return extensions.RenderContext{}, err From 9197ec92bb9bc4410df323c4d8b8df8cf4b67643 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Tue, 9 Jun 2026 11:19:14 -0700 Subject: [PATCH 19/38] Rename modCtx to renderCtx modCtx read like "modifier context"; the value is an extensions.RenderContext, so name it for what it is. --- pkg/controller/installation/core_controller.go | 6 +++--- pkg/controller/utils/component.go | 6 +++--- pkg/controller/utils/component_test.go | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pkg/controller/installation/core_controller.go b/pkg/controller/installation/core_controller.go index c173827d23..e4f32f7934 100644 --- a/pkg/controller/installation/core_controller.go +++ b/pkg/controller/installation/core_controller.go @@ -1219,7 +1219,7 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile // its variant - validating config and creating the node-prometheus // certificate, adding it (and the prometheus/esgw certs) to the trusted // bundle - and may abort the reconcile by returning an error. - modCtx, err := extensions.RunSetup(extensions.Inputs{ + renderCtx, err := extensions.RunSetup(extensions.Inputs{ Ctx: ctx, Client: r.client, Installation: &instance.Spec, @@ -1232,7 +1232,7 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile r.status.SetDegraded(operatorv1.ResourceCreateError, "Error preparing installation extension", err, reqLogger) return reconcile.Result{}, err } - nodePrometheusTLS := modCtx.NodePrometheusTLS + nodePrometheusTLS := renderCtx.NodePrometheusTLS kubeControllersMetricsPort, err := utils.GetKubeControllerMetricsPort(ctx, r.client) if err != nil { @@ -1271,7 +1271,7 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile } // Create a component handler to create or update the rendered components. - handler := r.newComponentHandler(log, r.client, r.scheme, instance, utils.WithRenderContext(modCtx)) + handler := r.newComponentHandler(log, r.client, r.scheme, instance, utils.WithRenderContext(renderCtx)) // Render namespaces first - this ensures that any other controllers blocked on namespace existence can proceed. namespaceCfg := &render.NamespaceConfiguration{ diff --git a/pkg/controller/utils/component.go b/pkg/controller/utils/component.go index 57238556da..550063296d 100644 --- a/pkg/controller/utils/component.go +++ b/pkg/controller/utils/component.go @@ -81,7 +81,7 @@ type ComponentHandlerOption func(*componentHandler) // WithRenderContext supplies the extensions.RenderContext passed to registered // render modifiers. func WithRenderContext(ctx extensions.RenderContext) ComponentHandlerOption { - return func(c *componentHandler) { c.modCtx = ctx } + return func(c *componentHandler) { c.renderCtx = ctx } } // cr is allowed to be nil in the case we don't want to put ownership on a resource, @@ -107,7 +107,7 @@ type componentHandler struct { log logr.Logger createOnly bool apiGroupEnvs []v1.EnvVar - modCtx extensions.RenderContext + renderCtx extensions.RenderContext } func (c *componentHandler) SetCreateOnly() { @@ -469,7 +469,7 @@ func (c *componentHandler) CreateOrUpdateOrDelete(ctx context.Context, component objsToCreate, objsToDelete := component.Objects() if ext, ok := component.(render.Extensible); ok { - objsToCreate = extensions.ApplyModifiers(ext.ModifierKey(), c.modCtx, objsToCreate) + objsToCreate = extensions.ApplyModifiers(ext.ModifierKey(), c.renderCtx, objsToCreate) } // Load the InstallationSpec once and reuse it for every object: createOrUpdateObject needs it diff --git a/pkg/controller/utils/component_test.go b/pkg/controller/utils/component_test.go index 93c888812d..ecbe90afe4 100644 --- a/pkg/controller/utils/component_test.go +++ b/pkg/controller/utils/component_test.go @@ -2506,8 +2506,8 @@ var _ = Describe("componentHandler modifier application", func() { Expect(corev1.SchemeBuilder.AddToScheme(s)).NotTo(HaveOccurred()) c := ctrlrfake.DefaultFakeClientBuilder(s).Build() - modCtx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}} - handler := NewComponentHandler(logf.Log, c, s, nil, WithRenderContext(modCtx)) + renderCtx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}} + handler := NewComponentHandler(logf.Log, c, s, nil, WithRenderContext(renderCtx)) comp := &namedFakeComponent{name: "fake", obj: &corev1.ConfigMap{ TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: "cm", Namespace: "default"}, From 6375a348d5ea318d97682bf6f486655c5b0eb850 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Tue, 9 Jun 2026 11:40:44 -0700 Subject: [PATCH 20/38] Document the extensions package and drop stale builder wording Add a package doc that lays out the two-phase model (Setup vs Extension) so the whole seam is legible from `go doc`. Fix two comments that still called the setup a render context builder. --- pkg/enterprise/node.go | 8 +++---- pkg/extensions/doc.go | 38 +++++++++++++++++++++++++++++++++ pkg/extensions/rendercontext.go | 8 +++---- 3 files changed, 46 insertions(+), 8 deletions(-) create mode 100644 pkg/extensions/doc.go diff --git a/pkg/enterprise/node.go b/pkg/enterprise/node.go index 335bac63c1..8e1de4832b 100644 --- a/pkg/enterprise/node.go +++ b/pkg/enterprise/node.go @@ -147,10 +147,10 @@ func modifyNodeDaemonSet(ctx extensions.RenderContext, ds *appsv1.DaemonSet) { // mountNodePrometheusTLS mounts the node prometheus reporter keypair onto the // daemonset: the volume, the calico-node volume mount, the cert-management init // container (when in use), and the pod hash annotation that rolls the pods on -// cert rotation. The keypair has cluster side effects, so the enterprise render -// context builder creates it and hands it in via ctx rather than the modifier -// building it. In core (calico) the keypair is never created, so the base node -// render carries no prometheus mount at all. +// cert rotation. The keypair has cluster side effects, so the enterprise setup +// creates it and hands it in via ctx rather than the modifier building it. In +// core (calico) the keypair is never created, so the base node render carries +// no prometheus mount at all. func mountNodePrometheusTLS(ctx extensions.RenderContext, ds *appsv1.DaemonSet) { if ctx.NodePrometheusTLS == nil { return diff --git a/pkg/extensions/doc.go b/pkg/extensions/doc.go new file mode 100644 index 0000000000..73adc07fa0 --- /dev/null +++ b/pkg/extensions/doc.go @@ -0,0 +1,38 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package extensions is the seam other product variants (today just Calico +// Enterprise) use to layer variant-specific behavior onto the core operator's +// render output, so core code never branches on variant. +// +// Everything keys off the installation Variant, and registration is per +// variant, so a registered hook only ever runs for its own variant and never +// re-checks it. There are two phases: +// +// Setup is the controller-side phase. It runs once per reconcile in the +// installation controller, has cluster access (Client, CertificateManager), and +// does the side-effecting work a pure render hook can't: creating certificates, +// extending the trusted bundle, validating config. It returns the RenderContext +// - the read-only baton passed to the render phase. Register one per variant +// with RegisterSetup; the controller runs it with RunSetup. +// +// Extension is the render phase: pure, per-component hooks that run after a +// component builds its objects. Its Image field overrides the component's image +// (resolved during ResolveImages), and its Modify field post-processes the +// rendered objects (run at the componentHandler). Register one per component +// with Register. +// +// A variant wires up its setup and extensions in one place at startup - see +// pkg/enterprise. +package extensions diff --git a/pkg/extensions/rendercontext.go b/pkg/extensions/rendercontext.go index 9ea25e1b6b..9653825a59 100644 --- a/pkg/extensions/rendercontext.go +++ b/pkg/extensions/rendercontext.go @@ -36,9 +36,9 @@ type RenderContext struct { // TrustedBundle is the shared CA bundle for the calico-system namespace. TrustedBundle certificatemanagement.TrustedBundle - // NodePrometheusTLS is created by the enterprise render context builder (it - // has cluster side effects, so it can't be built in a modifier). The node - // modifier is its only consumer: it mounts the keypair onto the daemonset and - // sets the FELIX_PROMETHEUSREPORTER* certificate env vars. + // NodePrometheusTLS is created by the enterprise setup (it has cluster side + // effects, so it can't be built in a modifier). The node modifier is its only + // consumer: it mounts the keypair onto the daemonset and sets the + // FELIX_PROMETHEUSREPORTER* certificate env vars. NodePrometheusTLS certificatemanagement.KeyPairInterface } From 147ec15feebbc538d5a2f72ad4d33612f974ff3a Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Tue, 9 Jun 2026 14:23:53 -0700 Subject: [PATCH 21/38] De-variant windows; add per-component context to extensions Add a per-component context channel: a component implements render.ExtensionContextProvider to hand its modifier config a modifier can't derive from the shared RenderContext (config only the component's controller has). The componentHandler reads it into RenderContext.Component before applying the modifier. node's setup-produced keypair keeps its own field; this is for component-config-derived inputs. Move windows's enterprise branches into a pkg/enterprise extension: the two windows image overrides, the node-metrics Service, the calico log volume (swapped in for the OSS cni-log mount), the enterprise felix env, the trusted DNS servers for openshift/rke2, and the prometheus reporter keypair mount. The windows component exposes its reporter port, keypair, and trusted bundle via ExtensionContext; the windows controller wires the render context into its handler. Core windows render is now OSS-only. --- .../installation/windows_controller.go | 6 +- pkg/controller/utils/component.go | 6 +- pkg/enterprise/register.go | 1 + pkg/enterprise/windows.go | 189 ++++++++++++++++++ pkg/enterprise/windows_test.go | 160 +++++++++++++++ pkg/extensions/rendercontext.go | 16 +- pkg/render/component.go | 17 ++ pkg/render/render_test.go | 14 +- pkg/render/windows.go | 147 +++----------- pkg/render/windows_test.go | 42 ++-- 10 files changed, 450 insertions(+), 148 deletions(-) create mode 100644 pkg/enterprise/windows.go create mode 100644 pkg/enterprise/windows_test.go diff --git a/pkg/controller/installation/windows_controller.go b/pkg/controller/installation/windows_controller.go index e23bf55db7..e3cb1389b2 100644 --- a/pkg/controller/installation/windows_controller.go +++ b/pkg/controller/installation/windows_controller.go @@ -51,6 +51,7 @@ import ( "github.com/tigera/operator/pkg/controller/utils/imageset" "github.com/tigera/operator/pkg/ctrlruntime" "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" "github.com/tigera/operator/pkg/render/monitor" "github.com/tigera/operator/pkg/tls/certificatemanagement" @@ -406,7 +407,10 @@ func (r *ReconcileWindows) Reconcile(ctx context.Context, request reconcile.Requ } // Create a component handler to create or update the rendered components. - handler := utils.NewComponentHandler(logw, r.client, r.scheme, instance) + // The render context carries the installation so registered modifiers gate on + // variant; the windows component supplies its own per-component context (the + // reporter port and prometheus keypair) via ExtensionContext. + handler := utils.NewComponentHandler(logw, r.client, r.scheme, instance, utils.WithRenderContext(extensions.RenderContext{Installation: &instance.Spec})) if err := handler.CreateOrUpdateOrDelete(ctx, component, nil); err != nil { r.status.SetDegraded(operatorv1.ResourceUpdateError, "Error creating / updating resource", err, reqLogger) return reconcile.Result{}, err diff --git a/pkg/controller/utils/component.go b/pkg/controller/utils/component.go index 550063296d..d2424c99c7 100644 --- a/pkg/controller/utils/component.go +++ b/pkg/controller/utils/component.go @@ -469,7 +469,11 @@ func (c *componentHandler) CreateOrUpdateOrDelete(ctx context.Context, component objsToCreate, objsToDelete := component.Objects() if ext, ok := component.(render.Extensible); ok { - objsToCreate = extensions.ApplyModifiers(ext.ModifierKey(), c.renderCtx, objsToCreate) + rc := c.renderCtx + if p, ok := component.(render.ExtensionContextProvider); ok { + rc.Component = p.ExtensionContext() + } + objsToCreate = extensions.ApplyModifiers(ext.ModifierKey(), rc, objsToCreate) } // Load the InstallationSpec once and reuse it for every object: createOrUpdateObject needs it diff --git a/pkg/enterprise/register.go b/pkg/enterprise/register.go index a01c9b7cfa..761f1736b7 100644 --- a/pkg/enterprise/register.go +++ b/pkg/enterprise/register.go @@ -20,5 +20,6 @@ package enterprise func Register() { registerTypha() registerNode() + registerWindows() registerInstallation() } diff --git a/pkg/enterprise/windows.go b/pkg/enterprise/windows.go new file mode 100644 index 0000000000..12f62cd6c6 --- /dev/null +++ b/pkg/enterprise/windows.go @@ -0,0 +1,189 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enterprise + +import ( + "fmt" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/client" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/render" + rmeta "github.com/tigera/operator/pkg/render/common/meta" +) + +// windowsNodeContainers are the calico-node-windows containers that share the +// felix env and node volume mounts, so they receive the same enterprise layering. +var windowsNodeContainers = map[string]bool{"felix": true, "node": true, "confd": true} + +func registerWindows() { + extensions.Register(operatorv1.CalicoEnterprise, render.ComponentNameWindowsNodeImg, extensions.Extension{ + Image: func(*operatorv1.InstallationSpec) components.Component { return components.ComponentTigeraNodeWindows }, + }) + extensions.Register(operatorv1.CalicoEnterprise, render.ComponentNameWindowsCNIImg, extensions.Extension{ + Image: func(*operatorv1.InstallationSpec) components.Component { return components.ComponentTigeraCNIWindows }, + }) + extensions.Register(operatorv1.CalicoEnterprise, render.ComponentNameWindows, extensions.Extension{ + Modify: modifyWindows, + }) +} + +// modifyWindows layers Calico Enterprise behavior onto the rendered +// calico-node-windows objects: the node-metrics Service and the Enterprise +// daemonset configuration (flow/DNS log env, prometheus reporter, trusted DNS +// servers, the calico log volume, and the prometheus reporter keypair mount). +func modifyWindows(ctx extensions.RenderContext, objs []client.Object) []client.Object { + wc, _ := ctx.Component.(render.WindowsExtensionContext) + + if ds, ok := extensions.FindObject[*appsv1.DaemonSet](objs, common.WindowsDaemonSetName); ok { + modifyWindowsDaemonSet(ctx, wc, ds) + } + + return append(objs, windowsNodeMetricsService(wc)) +} + +func modifyWindowsDaemonSet(ctx extensions.RenderContext, wc render.WindowsExtensionContext, ds *appsv1.DaemonSet) { + dirOrCreate := corev1.HostPathDirectoryOrCreate + spec := &ds.Spec.Template.Spec + + spec.Volumes = append(spec.Volumes, corev1.Volume{ + Name: "var-log-calico", + VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}, + }) + + for i := range spec.Containers { + c := &spec.Containers[i] + if !windowsNodeContainers[c.Name] { + continue + } + + c.Env = append(c.Env, windowsEnterpriseEnv(ctx, wc)...) + + // Enterprise mounts the calico log directory in place of the OSS CNI log + // directory, so drop the OSS mount before adding the enterprise one. + c.VolumeMounts = removeVolumeMount(c.VolumeMounts, "cni-log-dir") + c.VolumeMounts = append(c.VolumeMounts, corev1.VolumeMount{MountPath: "/var/log/calico", Name: "var-log-calico"}) + } + + mountWindowsPrometheusTLS(wc, ds) +} + +// windowsEnterpriseEnv is the Enterprise felix configuration added to the +// calico-node-windows containers. +func windowsEnterpriseEnv(ctx extensions.RenderContext, wc render.WindowsExtensionContext) []corev1.EnvVar { + env := []corev1.EnvVar{ + {Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, + {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: fmt.Sprintf("%d", wc.NodeReporterMetricsPort)}, + {Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, + {Name: "FELIX_FLOWLOGSFILEINCLUDELABELS", Value: "true"}, + {Name: "FELIX_FLOWLOGSFILEINCLUDEPOLICIES", Value: "true"}, + {Name: "FELIX_FLOWLOGSFILEINCLUDESERVICE", Value: "true"}, + {Name: "FELIX_FLOWLOGSENABLENETWORKSETS", Value: "true"}, + {Name: "FELIX_FLOWLOGSCOLLECTPROCESSINFO", Value: "true"}, + {Name: "FELIX_DNSLOGSFILEENABLED", Value: "true"}, + {Name: "FELIX_DNSLOGSFILEPERNODELIMIT", Value: "1000"}, + } + + if wc.PrometheusServerTLS != nil && wc.TrustedBundle != nil { + env = append(env, + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERCERTFILE", Value: wc.PrometheusServerTLS.VolumeMountCertificateFilePath()}, + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERKEYFILE", Value: wc.PrometheusServerTLS.VolumeMountKeyFilePath()}, + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERCAFILE", Value: wc.TrustedBundle.MountPath()}, + ) + } + + // Providers without a kube-dns service need a non-default trusted DNS server. + switch ctx.Installation.KubernetesProvider { + case operatorv1.ProviderOpenShift: + env = append(env, corev1.EnvVar{Name: "FELIX_DNSTRUSTEDSERVERS", Value: "k8s-service:openshift-dns/dns-default"}) + case operatorv1.ProviderRKE2: + env = append(env, corev1.EnvVar{Name: "FELIX_DNSTRUSTEDSERVERS", Value: "k8s-service:kube-system/rke2-coredns-rke2-coredns"}) + } + + return env +} + +// mountWindowsPrometheusTLS mounts the node prometheus reporter keypair onto the +// windows daemonset: the volume, the volume mount on each node container, and +// the pod hash annotation that rolls the pods on cert rotation. +func mountWindowsPrometheusTLS(wc render.WindowsExtensionContext, ds *appsv1.DaemonSet) { + if wc.PrometheusServerTLS == nil { + return + } + tls := wc.PrometheusServerTLS + spec := &ds.Spec.Template.Spec + + spec.Volumes = append(spec.Volumes, tls.Volume()) + + for i := range spec.Containers { + c := &spec.Containers[i] + if windowsNodeContainers[c.Name] { + c.VolumeMounts = append(c.VolumeMounts, tls.VolumeMount(rmeta.OSTypeWindows)) + } + } + + if ds.Spec.Template.Annotations == nil { + ds.Spec.Template.Annotations = map[string]string{} + } + ds.Spec.Template.Annotations[tls.HashAnnotationKey()] = tls.HashAnnotationValue() +} + +// windowsNodeMetricsService builds the enterprise-only calico-node-metrics-windows +// Service. +func windowsNodeMetricsService(wc render.WindowsExtensionContext) *corev1.Service { + return &corev1.Service{ + TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: render.WindowsNodeMetricsService, + Namespace: common.CalicoNamespace, + Labels: map[string]string{"k8s-app": render.WindowsNodeObjectName}, + }, + Spec: corev1.ServiceSpec{ + Selector: map[string]string{"k8s-app": render.WindowsNodeObjectName}, + ClusterIP: "None", + Ports: []corev1.ServicePort{ + { + Name: "calico-metrics-port", + Port: int32(wc.NodeReporterMetricsPort), + TargetPort: intstr.FromInt(wc.NodeReporterMetricsPort), + Protocol: corev1.ProtocolTCP, + }, + { + Name: "calico-bgp-metrics-port", + Port: render.NodeBGPReporterPort, + TargetPort: intstr.FromInt(int(render.NodeBGPReporterPort)), + Protocol: corev1.ProtocolTCP, + }, + }, + }, + } +} + +func removeVolumeMount(mounts []corev1.VolumeMount, name string) []corev1.VolumeMount { + out := mounts[:0] + for _, m := range mounts { + if m.Name != name { + out = append(out, m) + } + } + return out +} diff --git a/pkg/enterprise/windows_test.go b/pkg/enterprise/windows_test.go new file mode 100644 index 0000000000..b289941bdb --- /dev/null +++ b/pkg/enterprise/windows_test.go @@ -0,0 +1,160 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enterprise_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + client "sigs.k8s.io/controller-runtime/pkg/client" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/apis" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/controller/certificatemanager" + ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/enterprise" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/tls/certificatemanagement" +) + +var _ = Describe("windows enterprise image override", func() { + BeforeEach(func() { enterprise.Register() }) + AfterEach(func() { extensions.ResetForTest() }) + + ent := &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise} + calico := &operatorv1.InstallationSpec{Variant: operatorv1.Calico} + + It("selects the enterprise windows images for the enterprise variant", func() { + Expect(extensions.ResolveImage(render.ComponentNameWindowsNodeImg, components.ComponentCalicoNodeWindows, ent)).To(Equal(components.ComponentTigeraNodeWindows)) + Expect(extensions.ResolveImage(render.ComponentNameWindowsCNIImg, components.ComponentCalicoCNIWindows, ent)).To(Equal(components.ComponentTigeraCNIWindows)) + }) + + It("leaves the defaults in place for the Calico variant", func() { + Expect(extensions.ResolveImage(render.ComponentNameWindowsNodeImg, components.ComponentCalicoNodeWindows, calico)).To(Equal(components.ComponentCalicoNodeWindows)) + Expect(extensions.ResolveImage(render.ComponentNameWindowsCNIImg, components.ComponentCalicoCNIWindows, calico)).To(Equal(components.ComponentCalicoCNIWindows)) + }) +}) + +var _ = Describe("windows enterprise modifier", func() { + BeforeEach(func() { enterprise.Register() }) + AfterEach(func() { extensions.ResetForTest() }) + + // newObjs returns a windows daemonset with the node containers and the OSS + // cni-log-dir mount the modifier swaps out. + newObjs := func() []client.Object { + nodeContainer := func(name string) corev1.Container { + return corev1.Container{ + Name: name, + VolumeMounts: []corev1.VolumeMount{{MountPath: "/var/log/calico/cni", Name: "cni-log-dir"}}, + } + } + return []client.Object{ + &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{Name: common.WindowsDaemonSetName}, + Spec: appsv1.DaemonSetSpec{Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Containers: []corev1.Container{nodeContainer("felix"), nodeContainer("node"), nodeContainer("confd")}, + }}}, + }, + } + } + + ds := func(objs []client.Object) *appsv1.DaemonSet { + d, _ := extensions.FindObject[*appsv1.DaemonSet](objs, common.WindowsDaemonSetName) + return d + } + container := func(d *appsv1.DaemonSet, name string) *corev1.Container { + for i := range d.Spec.Template.Spec.Containers { + if d.Spec.Template.Spec.Containers[i].Name == name { + return &d.Spec.Template.Spec.Containers[i] + } + } + return nil + } + + ctxFor := func(provider operatorv1.Provider, tls certificatemanagement.KeyPairInterface, bundle certificatemanagement.TrustedBundleRO) extensions.RenderContext { + return extensions.RenderContext{ + Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise, KubernetesProvider: provider}, + Component: render.WindowsExtensionContext{ + NodeReporterMetricsPort: 9081, + PrometheusServerTLS: tls, + TrustedBundle: bundle, + }, + } + } + + It("appends the node-metrics service", func() { + out := extensions.ApplyModifiers(render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone, nil, nil), newObjs()) + svc, ok := extensions.FindObject[*corev1.Service](out, render.WindowsNodeMetricsService) + Expect(ok).To(BeTrue()) + Expect(svc.Namespace).To(Equal(common.CalicoNamespace)) + Expect(svc.Spec.Ports[0].Port).To(Equal(int32(9081))) + }) + + It("swaps the cni log mount for the calico log volume and adds enterprise env", func() { + out := extensions.ApplyModifiers(render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone, nil, nil), newObjs()) + d := ds(out) + + Expect(d.Spec.Template.Spec.Volumes).To(ContainElement(HaveField("Name", "var-log-calico"))) + for _, name := range []string{"felix", "node", "confd"} { + c := container(d, name) + Expect(c.VolumeMounts).To(ContainElement(HaveField("Name", "var-log-calico"))) + Expect(c.VolumeMounts).NotTo(ContainElement(HaveField("Name", "cni-log-dir"))) + Expect(c.Env).To(ContainElements( + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERPORT", Value: "9081"}, + corev1.EnvVar{Name: "FELIX_DNSLOGSFILEENABLED", Value: "true"}, + )) + } + }) + + It("sets the trusted DNS server on openshift", func() { + out := extensions.ApplyModifiers(render.ComponentNameWindows, ctxFor(operatorv1.ProviderOpenShift, nil, nil), newObjs()) + Expect(container(ds(out), "node").Env).To(ContainElement(corev1.EnvVar{Name: "FELIX_DNSTRUSTEDSERVERS", Value: "k8s-service:openshift-dns/dns-default"})) + }) + + It("mounts the prometheus reporter keypair when present", func() { + scheme := runtime.NewScheme() + Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) + cli := ctrlrfake.DefaultFakeClientBuilder(scheme).Build() + cm, err := certificatemanager.Create(cli, nil, "", common.OperatorNamespace(), certificatemanager.AllowCACreation()) + Expect(err).NotTo(HaveOccurred()) + tls, err := cm.GetOrCreateKeyPair(cli, render.NodePrometheusTLSServerSecret, common.OperatorNamespace(), []string{"calico-node-metrics-windows"}) + Expect(err).NotTo(HaveOccurred()) + bundle := cm.CreateTrustedBundle() + + out := extensions.ApplyModifiers(render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone, tls, bundle), newObjs()) + d := ds(out) + + Expect(d.Spec.Template.Spec.Volumes).To(ContainElement(tls.Volume())) + Expect(d.Spec.Template.Annotations).To(HaveKey(tls.HashAnnotationKey())) + Expect(container(d, "node").Env).To(ContainElement(HaveField("Name", "FELIX_PROMETHEUSREPORTERCERTFILE"))) + Expect(container(d, "node").VolumeMounts).To(ContainElement(tls.VolumeMount(render.Windows(&render.WindowsConfiguration{}).SupportedOSType()))) + }) + + It("does nothing for the Calico variant", func() { + ctx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}} + out := extensions.ApplyModifiers(render.ComponentNameWindows, ctx, newObjs()) + _, ok := extensions.FindObject[*corev1.Service](out, render.WindowsNodeMetricsService) + Expect(ok).To(BeFalse()) + Expect(ds(out).Spec.Template.Spec.Volumes).To(BeEmpty()) + }) +}) diff --git a/pkg/extensions/rendercontext.go b/pkg/extensions/rendercontext.go index 9653825a59..c03389c153 100644 --- a/pkg/extensions/rendercontext.go +++ b/pkg/extensions/rendercontext.go @@ -23,11 +23,13 @@ import ( // RenderContext carries reconcile-derived inputs from controllers into render // modifiers. Core operator code never reads these fields - only registered // modifiers do. -// Two kinds of value live here: +// Three kinds of value live here: // - raw cluster state gathered generically (Installation, FelixConfiguration, -// ClusterDomain) that modifiers derive their own values from, and +// ClusterDomain) that modifiers derive their own values from, // - controller-produced artifacts (TrustedBundle, NodePrometheusTLS) that can -// only be created controller-side because they have cluster side effects. +// only be created controller-side because they have cluster side effects, and +// - Component, the per-component context the component being modified supplies +// for config a modifier can't derive from the fields above. type RenderContext struct { Installation *operatorv1.InstallationSpec FelixConfiguration *v3.FelixConfiguration @@ -41,4 +43,12 @@ type RenderContext struct { // consumer: it mounts the keypair onto the daemonset and sets the // FELIX_PROMETHEUSREPORTER* certificate env vars. NodePrometheusTLS certificatemanagement.KeyPairInterface + + // Component is per-component context that the component being modified supplies + // via render.ExtensionContextProvider - config a modifier needs but can't + // derive from the fields above (e.g. a keypair the component's own controller + // created, or a CR field only that controller reads). The componentHandler + // sets it per component before applying the modifier; a modifier type-asserts + // it to the component's own context type. Nil when the component supplies none. + Component any } diff --git a/pkg/render/component.go b/pkg/render/component.go index 7785834dcf..3063e7f842 100644 --- a/pkg/render/component.go +++ b/pkg/render/component.go @@ -49,9 +49,26 @@ type Extensible interface { ModifierKey() string } +// ExtensionContextProvider is an optional companion to Extensible. A component +// implements it to hand its modifier component-specific context that can't be +// derived from the shared extensions.RenderContext - config only the component's +// controller has, such as a keypair the controller created. The componentHandler +// reads the returned value into RenderContext.Component before applying the +// modifier, and the modifier type-asserts it to the component's own context type. +type ExtensionContextProvider interface { + ExtensionContext() any +} + // Component names used as keys into the extension modifier registry. Keep these // in sync with the ModifierKey() methods that return them. const ( ComponentNameTypha = "typha" ComponentNameNode = "node" + + // ComponentNameWindows keys the windows daemonset modifier. The two windows + // images resolve through their own override keys, since one component renders + // both. + ComponentNameWindows = "windows" + ComponentNameWindowsNodeImg = "windows-node-image" + ComponentNameWindowsCNIImg = "windows-cni-image" ) diff --git a/pkg/render/render_test.go b/pkg/render/render_test.go index a172ce4f00..242c821bb2 100644 --- a/pkg/render/render_test.go +++ b/pkg/render/render_test.go @@ -224,16 +224,16 @@ var _ = Describe("Rendering tests", func() { }) It("should render all resources when variant is Tigera Secure", func() { - // For this scenario, we expect the basic resources plus the following for Tigera Secure: - // - X Same as default config - // - 1 Service to expose calico/node metrics. - // - 1 Service to expose Windows calico/node metrics. + // For this scenario, we expect the basic resources plus the following for Tigera Secure. + // The calico/node and Windows calico/node metrics Services are added by the + // enterprise modifiers at the componentHandler, not by Objects(), so they do + // not appear in this render-only aggregation. var nodeMetricsPort int32 = 9081 instance.Variant = operatorv1.CalicoEnterprise instance.NodeMetricsPort = &nodeMetricsPort c, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 9094, 0, nil, nil) Expect(err).To(BeNil(), "Expected Calico to create successfully %s", err) - Expect(componentCount(c)).To(Equal((5 + 3 + 4 + 1 + 6 + 6 + 1 + 2) + 1)) + Expect(componentCount(c)).To(Equal(5 + 3 + 4 + 1 + 6 + 6 + 1 + 2)) }) It("should render all resources when variant is Tigera Secure and Management Cluster", func() { @@ -277,8 +277,8 @@ var _ = Describe("Rendering tests", func() { &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: common.KubeControllersDeploymentName, Namespace: common.CalicoNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}}, &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "calico-kube-controllers-metrics", Namespace: common.CalicoNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}}, - // Windows node objects. - &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: render.WindowsNodeMetricsService, Namespace: common.CalicoNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}}, + // Windows node objects. The Windows node-metrics Service is added by the + // enterprise modifier at the componentHandler, so it is not in this output. &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cni-config-windows", Namespace: common.CalicoNamespace}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, &appsv1.DaemonSet{ObjectMeta: metav1.ObjectMeta{Name: common.WindowsDaemonSetName, Namespace: common.CalicoNamespace}, TypeMeta: metav1.TypeMeta{Kind: "DaemonSet", APIVersion: "apps/v1"}}, diff --git a/pkg/render/windows.go b/pkg/render/windows.go index c8af5dc762..15f388ecc8 100644 --- a/pkg/render/windows.go +++ b/pkg/render/windows.go @@ -24,13 +24,13 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" "sigs.k8s.io/controller-runtime/pkg/client" operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/controller/k8sapi" + "github.com/tigera/operator/pkg/imageoverride" rcomp "github.com/tigera/operator/pkg/render/common/components" rmeta "github.com/tigera/operator/pkg/render/common/meta" "github.com/tigera/operator/pkg/render/common/securitycontext" @@ -77,13 +77,10 @@ func (c *windowsComponent) ResolveImages(is *operatorv1.ImageSet) error { return imageName } - if c.cfg.Installation.Variant.IsEnterprise() { - c.cniImage = appendIfErr(components.GetReference(components.ComponentTigeraCNIWindows, reg, path, prefix, is)) - c.nodeImage = appendIfErr(components.GetReference(components.ComponentTigeraNodeWindows, reg, path, prefix, is)) - } else { - c.cniImage = appendIfErr(components.GetReference(components.ComponentCalicoCNIWindows, reg, path, prefix, is)) - c.nodeImage = appendIfErr(components.GetReference(components.ComponentCalicoNodeWindows, reg, path, prefix, is)) - } + cniImage := imageoverride.Resolve(ComponentNameWindowsCNIImg, components.ComponentCalicoCNIWindows, c.cfg.Installation) + nodeImage := imageoverride.Resolve(ComponentNameWindowsNodeImg, components.ComponentCalicoNodeWindows, c.cfg.Installation) + c.cniImage = appendIfErr(components.GetReference(cniImage, reg, path, prefix, is)) + c.nodeImage = appendIfErr(components.GetReference(nodeImage, reg, path, prefix, is)) if len(errMsgs) != 0 { return fmt.Errorf("%s", strings.Join(errMsgs, ",")) @@ -95,6 +92,27 @@ func (c *windowsComponent) SupportedOSType() rmeta.OSType { return rmeta.OSTypeWindows } +func (c *windowsComponent) ModifierKey() string { return ComponentNameWindows } + +// WindowsExtensionContext is the per-component context the windows modifier +// reads (via RenderContext.Component). It carries the enterprise inputs the +// windows controller has but a modifier can't derive from the installation: the +// reporter metrics port, the node prometheus keypair, and the trusted bundle +// the cert env vars reference. +type WindowsExtensionContext struct { + NodeReporterMetricsPort int + PrometheusServerTLS certificatemanagement.KeyPairInterface + TrustedBundle certificatemanagement.TrustedBundleRO +} + +func (c *windowsComponent) ExtensionContext() any { + return WindowsExtensionContext{ + NodeReporterMetricsPort: c.cfg.NodeReporterMetricsPort, + PrometheusServerTLS: c.cfg.PrometheusServerTLS, + TrustedBundle: c.cfg.TLS.TrustedBundle, + } +} + func (c *windowsComponent) Objects() ([]client.Object, []client.Object) { // Clean up old windows upgrader daemonset if present objsToDelete := []client.Object{ @@ -116,11 +134,6 @@ func (c *windowsComponent) Objects() ([]client.Object, []client.Object) { objs := []client.Object{} - if c.cfg.Installation.Variant.IsEnterprise() { - // Include Service for exposing node metrics. - objs = append(objs, c.nodeMetricsService()) - } - cniConfig := c.windowsCNIConfigMap() if cniConfig != nil { objs = append(objs, cniConfig) @@ -135,43 +148,6 @@ func (c *windowsComponent) Ready() bool { return true } -// nodeMetricsService creates a Service which exposes two endpoints on calico/node for -// reporting Prometheus metrics (for policy enforcement activity and BGP stats). -// This service is used internally by Calico Enterprise and is separate from general -// Prometheus metrics which are user-configurable. -func (c *windowsComponent) nodeMetricsService() *corev1.Service { - return &corev1.Service{ - TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: WindowsNodeMetricsService, - Namespace: common.CalicoNamespace, - Labels: map[string]string{"k8s-app": WindowsNodeObjectName}, - }, - Spec: corev1.ServiceSpec{ - Selector: map[string]string{"k8s-app": WindowsNodeObjectName}, - // Important: "None" tells Kubernetes that we want a headless service with - // no kube-proxy load balancer. If we omit this then kube-proxy will render - // a huge set of iptables rules for this service since there's an instance - // on every node. - ClusterIP: "None", - Ports: []corev1.ServicePort{ - { - Name: "calico-metrics-port", - Port: int32(c.cfg.NodeReporterMetricsPort), - TargetPort: intstr.FromInt(c.cfg.NodeReporterMetricsPort), - Protocol: corev1.ProtocolTCP, - }, - { - Name: "calico-bgp-metrics-port", - Port: NodeBGPReporterPort, - TargetPort: intstr.FromInt(int(NodeBGPReporterPort)), - Protocol: corev1.ProtocolTCP, - }, - }, - }, - } -} - // windowsCNIConfigMap returns a config map containing the CNI network config to be installed on each node. // Returns nil if no configmap is needed. func (c *windowsComponent) windowsCNIConfigMap() *corev1.ConfigMap { @@ -380,8 +356,8 @@ func (c *windowsComponent) windowsVolumes() []corev1.Volume { {Name: "policysync", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/nodeagent", Type: &dirOrCreate}}}, c.cfg.TLS.TrustedBundle.Volume(), c.cfg.TLS.NodeSecret.Volume(), - corev1.Volume{Name: "var-run-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/calico", Type: &dirOrCreate}}}, - corev1.Volume{Name: "var-lib-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/lib/calico", Type: &dirOrCreate}}}, + {Name: "var-run-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/calico", Type: &dirOrCreate}}}, + {Name: "var-lib-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/lib/calico", Type: &dirOrCreate}}}, } // If needed for this configuration, then include the CNI volumes. @@ -392,20 +368,6 @@ func (c *windowsComponent) windowsVolumes() []corev1.Volume { volumes = append(volumes, corev1.Volume{Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: c.cfg.Installation.WindowsNodes.CNILogDir, Type: &dirOrCreate}}}) } - // Override with Tigera-specific config. - if c.cfg.Installation.Variant.IsEnterprise() { - // Add volume for calico logs. - calicoLogVol := corev1.Volume{ - Name: "var-log-calico", - VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}, - } - volumes = append(volumes, calicoLogVol) - } - - if c.cfg.PrometheusServerTLS != nil { - volumes = append(volumes, c.cfg.PrometheusServerTLS.Volume()) - } - return volumes } @@ -479,7 +441,6 @@ func (c *windowsComponent) nodeContainer() corev1.Container { // felixContainer creates the windows felix container. func (c *windowsComponent) felixContainer() corev1.Container { - lp, rp := c.windowsLivenessReadinessProbes() return corev1.Container{ @@ -659,31 +620,6 @@ func (c *windowsComponent) windowsEnvVars() []corev1.EnvVar { windowsEnv = append(windowsEnv, corev1.EnvVar{Name: "FELIX_IPV6SUPPORT", Value: "false"}) } - if c.cfg.Installation.Variant.IsEnterprise() { - // Add in Calico Enterprise specific configuration. - extraNodeEnv := []corev1.EnvVar{ - {Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, - {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: fmt.Sprintf("%d", c.cfg.NodeReporterMetricsPort)}, - {Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDELABELS", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDEPOLICIES", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDESERVICE", Value: "true"}, - {Name: "FELIX_FLOWLOGSENABLENETWORKSETS", Value: "true"}, - {Name: "FELIX_FLOWLOGSCOLLECTPROCESSINFO", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEPERNODELIMIT", Value: "1000"}, - } - - if c.cfg.PrometheusServerTLS != nil { - extraNodeEnv = append(extraNodeEnv, - corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERCERTFILE", Value: c.cfg.PrometheusServerTLS.VolumeMountCertificateFilePath()}, - corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERKEYFILE", Value: c.cfg.PrometheusServerTLS.VolumeMountKeyFilePath()}, - corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERCAFILE", Value: c.cfg.TLS.TrustedBundle.MountPath()}, - ) - } - windowsEnv = append(windowsEnv, extraNodeEnv...) - } - if c.cfg.Installation.NodeMetricsPort != nil { // If a node metrics port was given, then enable felix prometheus metrics and set the port. // Note that this takes precedence over any FelixConfiguration resources in the cluster. @@ -694,20 +630,6 @@ func (c *windowsComponent) windowsEnvVars() []corev1.EnvVar { windowsEnv = append(windowsEnv, extraNodeEnv...) } - // Configure provider specific environment variables here. - switch c.cfg.Installation.KubernetesProvider { - case operatorv1.ProviderOpenShift: - if c.cfg.Installation.Variant.IsEnterprise() { - // We need to configure a non-default trusted DNS server, since there's no kube-dns. - windowsEnv = append(windowsEnv, corev1.EnvVar{Name: "FELIX_DNSTRUSTEDSERVERS", Value: "k8s-service:openshift-dns/dns-default"}) - } - case operatorv1.ProviderRKE2: - // For RKE2, configure a non-default trusted DNS server, as the DNS service is not named "kube-dns". - if c.cfg.Installation.Variant.IsEnterprise() { - windowsEnv = append(windowsEnv, corev1.EnvVar{Name: "FELIX_DNSTRUSTEDSERVERS", Value: "k8s-service:kube-system/rke2-coredns-rke2-coredns"}) - } - } - if c.cfg.Installation.CNI.Type != operatorv1.PluginCalico { windowsEnv = append(windowsEnv, corev1.EnvVar{Name: "FELIX_ROUTESOURCE", Value: "WorkloadIPs"}) } @@ -726,12 +648,7 @@ func (c *windowsComponent) windowsVolumeMounts() []corev1.VolumeMount { corev1.VolumeMount{MountPath: "/var/run/calico", Name: "var-run-calico"}, corev1.VolumeMount{MountPath: "/var/lib/calico", Name: "var-lib-calico"}) - if c.cfg.Installation.Variant.IsEnterprise() { - extraNodeMounts := []corev1.VolumeMount{ - {MountPath: "/var/log/calico", Name: "var-log-calico"}, - } - windowsVolumeMounts = append(windowsVolumeMounts, extraNodeMounts...) - } else if c.cfg.Installation.CNI.Type == operatorv1.PluginCalico { + if c.cfg.Installation.CNI.Type == operatorv1.PluginCalico { windowsVolumeMounts = append(windowsVolumeMounts, corev1.VolumeMount{MountPath: "/var/log/calico/cni", Name: "cni-log-dir", ReadOnly: false}) } @@ -739,9 +656,6 @@ func (c *windowsComponent) windowsVolumeMounts() []corev1.VolumeMount { windowsVolumeMounts = append(windowsVolumeMounts, corev1.VolumeMount{MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}) } - if c.cfg.PrometheusServerTLS != nil { - windowsVolumeMounts = append(windowsVolumeMounts, c.cfg.PrometheusServerTLS.VolumeMount(c.SupportedOSType())) - } return windowsVolumeMounts } @@ -797,9 +711,6 @@ func (c *windowsComponent) windowsDaemonset(cniCfgMap *corev1.ConfigMap) *appsv1 initContainers := []corev1.Container{c.uninstallContainer()} annotations := c.cfg.TLS.TrustedBundle.HashAnnotations() - if c.cfg.PrometheusServerTLS != nil { - annotations[c.cfg.PrometheusServerTLS.HashAnnotationKey()] = c.cfg.PrometheusServerTLS.HashAnnotationValue() - } if cniCfgMap != nil { annotations[nodeCniConfigAnnotation] = rmeta.AnnotationHash(cniCfgMap.Data) diff --git a/pkg/render/windows_test.go b/pkg/render/windows_test.go index ebb1c4d9b1..1eb52c0849 100644 --- a/pkg/render/windows_test.go +++ b/pkg/render/windows_test.go @@ -35,12 +35,28 @@ import ( "github.com/tigera/operator/pkg/controller/certificatemanager" "github.com/tigera/operator/pkg/controller/k8sapi" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" rmeta "github.com/tigera/operator/pkg/render/common/meta" rtest "github.com/tigera/operator/pkg/render/common/test" "github.com/tigera/operator/pkg/tls/certificatemanagement" ) +// renderWindows renders the windows component and applies the registered +// enterprise modifier the way the componentHandler does, so enterprise tests +// exercise the integrated output (image overrides come from ResolveImages; the +// metrics service, env, volumes and mounts come from the modifier). +func renderWindows(cfg *render.WindowsConfiguration) []client.Object { + comp := render.Windows(cfg) + ExpectWithOffset(1, comp.ResolveImages(nil)).To(BeNil()) + objs, _ := comp.Objects() + rc := extensions.RenderContext{Installation: cfg.Installation} + if p, ok := comp.(render.ExtensionContextProvider); ok { + rc.Component = p.ExtensionContext() + } + return extensions.ApplyModifiers(render.ComponentNameWindows, rc, objs) +} + var _ = Describe("Windows rendering tests", func() { var defaultInstance *operatorv1.InstallationSpec var typhaNodeTLS *render.TyphaNodeTLS @@ -694,16 +710,14 @@ var _ = Describe("Windows rendering tests", func() { version string kind string }{ - {name: "calico-node-metrics-windows", ns: "calico-system", group: "", version: "v1", kind: "Service"}, {name: "cni-config-windows", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ConfigMap"}, {name: common.WindowsDaemonSetName, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "DaemonSet"}, + {name: "calico-node-metrics-windows", ns: "calico-system", group: "", version: "v1", kind: "Service"}, } defaultInstance.Variant = operatorv1.CalicoEnterprise cfg.NodeReporterMetricsPort = 9081 - component := render.Windows(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources := renderWindows(&cfg) Expect(len(resources)).To(Equal(len(expectedResources))) // Should render the correct resources. @@ -1686,18 +1700,16 @@ var _ = Describe("Windows rendering tests", func() { version string kind string }{ - {name: "calico-node-metrics-windows", ns: "calico-system", group: "", version: "v1", kind: "Service"}, {name: "cni-config-windows", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ConfigMap"}, {name: common.WindowsDaemonSetName, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "DaemonSet"}, + {name: "calico-node-metrics-windows", ns: "calico-system", group: "", version: "v1", kind: "Service"}, } defaultInstance.Variant = operatorv1.CalicoEnterprise defaultInstance.KubernetesProvider = operatorv1.ProviderOpenShift cfg.NodeReporterMetricsPort = 9081 - component := render.Windows(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources := renderWindows(&cfg) Expect(len(resources)).To(Equal(len(expectedResources))) // Should render the correct resources. @@ -1842,18 +1854,16 @@ var _ = Describe("Windows rendering tests", func() { version string kind string }{ - {name: "calico-node-metrics-windows", ns: "calico-system", group: "", version: "v1", kind: "Service"}, {name: "cni-config-windows", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ConfigMap"}, {name: common.WindowsDaemonSetName, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "DaemonSet"}, + {name: "calico-node-metrics-windows", ns: "calico-system", group: "", version: "v1", kind: "Service"}, } defaultInstance.Variant = operatorv1.CalicoEnterprise defaultInstance.KubernetesProvider = operatorv1.ProviderRKE2 cfg.NodeReporterMetricsPort = 9081 - component := render.Windows(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources := renderWindows(&cfg) Expect(len(resources)).To(Equal(len(expectedResources)), fmt.Sprintf("Actual resources: %#v", resources)) // Should render the correct resources. @@ -2138,9 +2148,7 @@ var _ = Describe("Windows rendering tests", func() { defaultInstance.NodeMetricsPort = nil cfg.NodeReporterMetricsPort = 9081 - component := render.Windows(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources := renderWindows(&cfg) Expect(len(resources)).To(Equal(defaultNumExpectedResources + 1)) dsResource := rtest.GetResource(resources, "calico-node-windows", "calico-system", "apps", "v1", "DaemonSet") @@ -2159,9 +2167,7 @@ var _ = Describe("Windows rendering tests", func() { var nodeMetricsPort int32 = 1234 defaultInstance.Variant = operatorv1.CalicoEnterprise defaultInstance.NodeMetricsPort = &nodeMetricsPort - component := render.Windows(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources := renderWindows(&cfg) Expect(len(resources)).To(Equal(defaultNumExpectedResources + 1)) dsResource := rtest.GetResource(resources, "calico-node-windows", "calico-system", "apps", "v1", "DaemonSet") From ffe993e59db246b1aa8e97151ca0590161ed83a5 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Tue, 9 Jun 2026 15:21:14 -0700 Subject: [PATCH 22/38] De-variant the guardian component into an enterprise extension Move guardian's enterprise branches out of core render into a pkg/enterprise extension: the secrets Role/RoleBinding and default UI settings, the elasticsearch/kibana service ports, the management-cluster-request cluster role rules (which wholly replace the OSS rules, plus impersonation and the OpenShift SCC), and the CA bundle env vars. rulesForManagementClusterRequests moves to enterprise; the UI-settings helpers are exported from the manager render so the modifier can build them. The guardian component exposes its OpenShift flag, impersonation config, and trusted bundle path via ExtensionContext; the clusterconnection controller wires the render context into its handler. Core guardian render is now OSS-only except the network policy, which is the next commit. --- .../clusterconnection_controller.go | 6 +- .../clusterconnection_suite_test.go | 7 + pkg/enterprise/guardian.go | 485 ++++++++++++++++ pkg/enterprise/guardian_test.go | 109 ++++ pkg/enterprise/register.go | 1 + pkg/render/guardian.go | 529 ++---------------- pkg/render/guardian_test.go | 46 +- pkg/render/manager.go | 24 +- 8 files changed, 709 insertions(+), 498 deletions(-) create mode 100644 pkg/enterprise/guardian.go create mode 100644 pkg/enterprise/guardian_test.go diff --git a/pkg/controller/clusterconnection/clusterconnection_controller.go b/pkg/controller/clusterconnection/clusterconnection_controller.go index 4b1b7dc8f6..b46ad0c205 100644 --- a/pkg/controller/clusterconnection/clusterconnection_controller.go +++ b/pkg/controller/clusterconnection/clusterconnection_controller.go @@ -49,6 +49,7 @@ import ( "github.com/tigera/operator/pkg/controller/utils/imageset" "github.com/tigera/operator/pkg/ctrlruntime" "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" "github.com/tigera/operator/pkg/render/common/networkpolicy" "github.com/tigera/operator/pkg/render/goldmane" @@ -443,7 +444,10 @@ func (r *ReconcileConnection) Reconcile(ctx context.Context, request reconcile.R return reconcile.Result{}, err } - ch := utils.NewComponentHandler(log, r.cli, r.scheme, managementClusterConnection) + // The render context carries the installation so registered modifiers gate on + // variant; the guardian component supplies its own per-component context (the + // impersonation config, OpenShift, and CA bundle path) via ExtensionContext. + ch := utils.NewComponentHandler(log, r.cli, r.scheme, managementClusterConnection, utils.WithRenderContext(extensions.RenderContext{Installation: installationSpec})) guardianCfg := &render.GuardianConfiguration{ URL: managementClusterConnection.Spec.ManagementClusterAddr, PodProxies: r.resolvedPodProxies, diff --git a/pkg/controller/clusterconnection/clusterconnection_suite_test.go b/pkg/controller/clusterconnection/clusterconnection_suite_test.go index 8967498282..09a8254690 100644 --- a/pkg/controller/clusterconnection/clusterconnection_suite_test.go +++ b/pkg/controller/clusterconnection/clusterconnection_suite_test.go @@ -22,11 +22,18 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log/zap" logf "sigs.k8s.io/controller-runtime/pkg/log" + + "github.com/tigera/operator/pkg/enterprise" ) func TestStatus(t *testing.T) { logf.SetLogger(zap.New(zap.WriteTo(ginkgo.GinkgoWriter))) gomega.RegisterFailHandler(ginkgo.Fail) + + // Wire the enterprise extensions so the guardian modifier runs the way it + // does in the operator binary, which is what these controller tests exercise. + enterprise.Register() + suiteConfig, reporterConfig := ginkgo.GinkgoConfiguration() reporterConfig.JUnitReport = "../../../report/ut/clusterconnection_controller_suite.xml" ginkgo.RunSpecs(t, "pkg/controller/Management Cluster Connection Suite", suiteConfig, reporterConfig) diff --git a/pkg/enterprise/guardian.go b/pkg/enterprise/guardian.go new file mode 100644 index 0000000000..bcac8a48e1 --- /dev/null +++ b/pkg/enterprise/guardian.go @@ -0,0 +1,485 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enterprise + +import ( + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/client" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/render/common/securitycontextconstraints" +) + +func registerGuardian() { + extensions.Register(operatorv1.CalicoEnterprise, render.GuardianName, extensions.Extension{ + Modify: modifyGuardian, + }) +} + +// modifyGuardian layers Calico Enterprise behavior onto the rendered guardian +// objects: the secrets Role/RoleBinding and default UI settings, the +// elasticsearch/kibana service ports, the management-cluster-request cluster +// role rules (which replace the OSS rules), and the CA bundle env vars. +func modifyGuardian(ctx extensions.RenderContext, objs []client.Object) []client.Object { + gc, _ := ctx.Component.(render.GuardianExtensionContext) + + if role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, render.GuardianClusterRoleName); ok { + role.Rules = guardianEnterpriseRules(gc) + } + + if svc, ok := extensions.FindObject[*corev1.Service](objs, render.GuardianServiceName); ok { + svc.Spec.Ports = append(svc.Spec.Ports, guardianEnterpriseServicePorts()...) + } + + if dep, ok := extensions.FindObject[*appsv1.Deployment](objs, render.GuardianDeploymentName); ok { + addGuardianEnterpriseEnv(gc, dep) + } + + return append(objs, + guardianSecretsRole(), + guardianSecretRoleBinding(), + // Default UI settings for this managed cluster. + render.ManagerClusterWideSettingsGroup(), + render.ManagerUserSpecificSettingsGroup(), + render.ManagerClusterWideTigeraLayer(), + render.ManagerClusterWideDefaultView(), + ) +} + +// guardianEnterpriseRules are the cluster role rules guardian needs in Calico +// Enterprise. They wholly replace the OSS rules: the management cluster drives +// guardian over the tunnel, so it needs the union of the rules its components +// require, plus any configured impersonation and the OpenShift SCC. +func guardianEnterpriseRules(gc render.GuardianExtensionContext) []rbacv1.PolicyRule { + var rules []rbacv1.PolicyRule + + if imp := gc.Impersonation; imp != nil { + if imp.Users != nil { + rules = append(rules, rbacv1.PolicyRule{ + APIGroups: []string{""}, + Resources: []string{"users"}, + ResourceNames: imp.Users, + Verbs: []string{"impersonate"}, + }) + } + if imp.Groups != nil { + rules = append(rules, rbacv1.PolicyRule{ + APIGroups: []string{""}, + Resources: []string{"groups"}, + ResourceNames: imp.Groups, + Verbs: []string{"impersonate"}, + }) + } + if imp.ServiceAccounts != nil { + rules = append(rules, rbacv1.PolicyRule{ + APIGroups: []string{""}, + Resources: []string{"serviceaccounts"}, + ResourceNames: imp.ServiceAccounts, + Verbs: []string{"impersonate"}, + }) + } + } + + rules = append(rules, rulesForManagementClusterRequests(gc.OpenShift)...) + + if gc.OpenShift { + rules = append(rules, rbacv1.PolicyRule{ + APIGroups: []string{"security.openshift.io"}, + Resources: []string{"securitycontextconstraints"}, + Verbs: []string{"use"}, + ResourceNames: []string{securitycontextconstraints.NonRootV2}, + }) + } + + return rules +} + +func guardianEnterpriseServicePorts() []corev1.ServicePort { + return []corev1.ServicePort{ + { + Name: "elasticsearch", + Port: 9200, + TargetPort: intstr.IntOrString{Type: intstr.Int, IntVal: 8080}, + Protocol: corev1.ProtocolTCP, + }, + { + Name: "kibana", + Port: 5601, + TargetPort: intstr.IntOrString{Type: intstr.Int, IntVal: 8080}, + Protocol: corev1.ProtocolTCP, + }, + } +} + +func addGuardianEnterpriseEnv(gc render.GuardianExtensionContext, dep *appsv1.Deployment) { + for i := range dep.Spec.Template.Spec.Containers { + c := &dep.Spec.Template.Spec.Containers[i] + if c.Name != render.GuardianContainerName { + continue + } + c.Env = append(c.Env, + corev1.EnvVar{Name: "GUARDIAN_PACKET_CAPTURE_CA_BUNDLE_PATH", Value: gc.TrustedBundleMountPath}, + corev1.EnvVar{Name: "GUARDIAN_PROMETHEUS_CA_BUNDLE_PATH", Value: gc.TrustedBundleMountPath}, + corev1.EnvVar{Name: "GUARDIAN_QUERYSERVER_CA_BUNDLE_PATH", Value: gc.TrustedBundleMountPath}, + ) + } +} + +// guardianSecretsRole creates a Role that allows the management cluster to +// provision secrets to the tigera-operator Namespace, used to push secrets the +// managed cluster needs to access / authenticate with the management cluster. +func guardianSecretsRole() *rbacv1.Role { + return &rbacv1.Role{ + TypeMeta: metav1.TypeMeta{Kind: "Role", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: render.GuardianSecretsRole, + Namespace: common.OperatorNamespace(), + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{"secrets"}, + Verbs: []string{"create", "delete", "deletecollection", "update"}, + }, + }, + } +} + +func guardianSecretRoleBinding() *rbacv1.RoleBinding { + return &rbacv1.RoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: render.GuardianSecretsRoleBindingName, + Namespace: common.OperatorNamespace(), + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "Role", + Name: render.GuardianSecretsRole, + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: render.GuardianServiceAccountName, + Namespace: render.GuardianNamespace, + }, + }, + } +} + +// rulesForManagementClusterRequests returns the set of RBAC rules guardian needs +// to satisfy requests from the management cluster over the tunnel. +func rulesForManagementClusterRequests(isOpenShift bool) []rbacv1.PolicyRule { + rules := []rbacv1.PolicyRule{ + // Common rules required to handle requests from multiple components in the management cluster. + { + // ID uses read-only permissions and kube-controllers uses both read and write verbs. + APIGroups: []string{""}, + Resources: []string{"configmaps"}, + Verbs: []string{"create", "delete", "get", "list", "update", "watch"}, + }, + { + // Allows Linseed to watch namespaces before copying its token. + // Also enables PolicyRecommendation to watch namespaces, + // and Manager/kube-controllers to list them. + APIGroups: []string{""}, + Resources: []string{"namespaces"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + // kube-controllers watches Nodes to monitor for deletions. + // Manager performs a list operation on Nodes. + APIGroups: []string{""}, + Resources: []string{"nodes"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + // kube-controllers watches Pods to verify existence for IPAM garbage collection. + // Manager performs get operations on Pods. + APIGroups: []string{""}, + Resources: []string{"pods"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + // The Federated Services Controller needs access to the remote kubeconfig secret + // in order to create a remote syncer. + APIGroups: []string{""}, + Resources: []string{"secrets"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + // Manager uses list; kube-controllers uses 'get', 'list', 'watch', 'update'. + APIGroups: []string{""}, + Resources: []string{"services"}, + Verbs: []string{"get", "list", "update", "watch"}, + }, + { + // Needed by kube-controllers to validate licenses; also used by ID. + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"licensekeys"}, + Verbs: []string{"get", "watch"}, + }, + { + // Manager uses list; PolicyRecommendation & ID uses all verbs. + APIGroups: []string{"projectcalico.org"}, + Resources: []string{ + "globalnetworksets", + "networkpolicies", + "tier.networkpolicies", + "stagednetworkpolicies", + "tier.stagednetworkpolicies", + }, + Verbs: []string{"create", "delete", "get", "list", "patch", "update", "watch"}, + }, + { + // Manager uses list; PolicyRecommendation uses all verbs. + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"tiers"}, + Verbs: []string{"create", "delete", "get", "list", "patch", "update", "watch"}, + }, + // Rules needed by guardian to handle manager authorization reviews. + { + APIGroups: []string{"rbac.authorization.k8s.io"}, + Resources: []string{"clusterroles", "clusterrolebindings", "roles", "rolebindings"}, + Verbs: []string{"list", "get"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"uisettings", "uisettingsgroups"}, + Verbs: []string{"list", "get"}, + }, + + // Rules needed by guardian to handle other manager requests. + { + APIGroups: []string{""}, + Resources: []string{"events"}, + Verbs: []string{"list"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"serviceaccounts"}, + Verbs: []string{"list"}, + }, + { + // Allow query server talk to Prometheus via the manager user. + APIGroups: []string{""}, + Resources: []string{"services/proxy"}, + ResourceNames: []string{ + "calico-node-prometheus:9090", + "https:calico-api:8080", + }, + Verbs: []string{"create", "get"}, + }, + { + APIGroups: []string{"apps"}, + Resources: []string{"daemonsets", "replicasets", "statefulsets"}, + Verbs: []string{"list"}, + }, + { + APIGroups: []string{"authentication.k8s.io"}, + Resources: []string{"tokenreviews"}, + Verbs: []string{"create"}, + }, + { + APIGroups: []string{"authorization.k8s.io"}, + Resources: []string{"subjectaccessreviews"}, + Verbs: []string{"create"}, + }, + { + APIGroups: []string{"networking.k8s.io"}, + Resources: []string{"networkpolicies"}, + Verbs: []string{"get", "list"}, + }, + { + APIGroups: []string{"policy.networking.k8s.io"}, + Resources: []string{ + "clusternetworkpolicies", + "adminnetworkpolicies", + "baselineadminnetworkpolicies", + }, + Verbs: []string{"list"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"alertexceptions"}, + Verbs: []string{"get", "list", "update"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"felixconfigurations"}, + ResourceNames: []string{"default"}, + Verbs: []string{"get"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{ + "globalnetworkpolicies", + "networksets", + "stagedglobalnetworkpolicies", + "stagedkubernetesnetworkpolicies", + "tier.globalnetworkpolicies", + "tier.stagedglobalnetworkpolicies", + }, + Verbs: []string{"list"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"hostendpoints"}, + Verbs: []string{"list"}, + }, + + // Rules needed by guardian to handle policy recommendation requests. + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{ + "policyrecommendationscopes", + "policyrecommendationscopes/status", + }, + Verbs: []string{"create", "delete", "get", "list", "patch", "update", "watch"}, + }, + + // Rules needed by guardian to handle calico-kube-controller requests. + { + // Nodes are watched to monitor for deletions. + APIGroups: []string{""}, + Resources: []string{"endpoints"}, + Verbs: []string{"create", "delete", "get", "list", "update", "watch"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"services/status"}, + Verbs: []string{"get", "list", "update", "watch"}, + }, + { + // Needs to manage hostendpoints. + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"hostendpoints"}, + Verbs: []string{"create", "delete", "get", "list", "update", "watch"}, + }, + { + // Needs access to update clusterinformations. + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"clusterinformations"}, + Verbs: []string{"create", "get", "list", "update", "watch"}, + }, + { + // Needs to manipulate kubecontrollersconfiguration, which contains its config. + // It creates a default if none exists, and updates status as well. + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"kubecontrollersconfigurations"}, + Verbs: []string{"create", "get", "list", "update", "watch"}, + }, + { + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"tiers"}, + Verbs: []string{"create"}, + }, + { + APIGroups: []string{"crd.projectcalico.org", "projectcalico.org"}, + Resources: []string{"deeppacketinspections"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"deeppacketinspections/status"}, + Verbs: []string{"update"}, + }, + { + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"packetcaptures"}, + Verbs: []string{"get", "list", "update"}, + }, + { + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"remoteclusterconfigurations"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"licensekeys"}, + Verbs: []string{"create", "get", "list", "update", "watch"}, + }, + { + // Grant permissions to access ClusterInformation resources in managed clusters. + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"clusterinformations"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + APIGroups: []string{"usage.tigera.io"}, + Resources: []string{"licenseusagereports"}, + Verbs: []string{"create", "delete", "get", "list", "update", "watch"}, + }, + + // Rules needed by guardian to handle Intrusion detection requests. + { + APIGroups: []string{""}, + Resources: []string{"podtemplates"}, + Verbs: []string{"get"}, + }, + { + APIGroups: []string{"apps"}, + Resources: []string{"deployments"}, + Verbs: []string{"get"}, + }, + { + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"alertexceptions"}, + Verbs: []string{"get", "list"}, + }, + { + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"securityeventwebhooks"}, + Verbs: []string{"get", "list", "update", "watch"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{ + "globalalerts", + "globalalerts/status", + "globalthreatfeeds", + "globalthreatfeeds/status", + }, + Verbs: []string{"create", "delete", "get", "list", "patch", "update", "watch"}, + }, + // Rules needed to fetch the compliance reports + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"globalreporttypes", "globalreports"}, + Verbs: []string{"get", "list", "watch"}, + }, + } + + // Rules needed by policy recommendation in openshift. + if isOpenShift { + rules = append(rules, + rbacv1.PolicyRule{ + APIGroups: []string{"security.openshift.io"}, + Resources: []string{"securitycontextconstraints"}, + Verbs: []string{"use"}, + ResourceNames: []string{securitycontextconstraints.HostNetworkV2}, + }, + ) + } + + return rules +} diff --git a/pkg/enterprise/guardian_test.go b/pkg/enterprise/guardian_test.go new file mode 100644 index 0000000000..b68ad61b04 --- /dev/null +++ b/pkg/enterprise/guardian_test.go @@ -0,0 +1,109 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enterprise_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + client "sigs.k8s.io/controller-runtime/pkg/client" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/enterprise" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/render" +) + +var _ = Describe("guardian enterprise modifier", func() { + BeforeEach(func() { enterprise.Register() }) + AfterEach(func() { extensions.ResetForTest() }) + + // newObjs returns the subset of rendered guardian objects the modifier touches. + newObjs := func() []client.Object { + return []client.Object{ + &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: render.GuardianClusterRoleName}, Rules: []rbacv1.PolicyRule{{Verbs: []string{"get"}}}}, + &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{Name: render.GuardianServiceName, Namespace: render.GuardianNamespace}, + Spec: corev1.ServiceSpec{Ports: []corev1.ServicePort{{Name: "https", Port: 443}}}, + }, + &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: render.GuardianDeploymentName, Namespace: render.GuardianNamespace}, + Spec: appsv1.DeploymentSpec{Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: render.GuardianContainerName}}, + }}}, + }, + } + } + + ctxWith := func(c render.GuardianExtensionContext) extensions.RenderContext { + return extensions.RenderContext{ + Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}, + Component: c, + } + } + + It("appends the secrets RBAC and UI settings", func() { + out := extensions.ApplyModifiers(render.GuardianName, ctxWith(render.GuardianExtensionContext{}), newObjs()) + _, ok := extensions.FindObject[*rbacv1.Role](out, render.GuardianSecretsRole) + Expect(ok).To(BeTrue()) + _, ok = extensions.FindObject[*rbacv1.RoleBinding](out, render.GuardianSecretsRoleBindingName) + Expect(ok).To(BeTrue()) + _, ok = extensions.FindObject[*v3.UISettingsGroup](out, render.ManagerClusterSettings) + Expect(ok).To(BeTrue()) + }) + + It("adds the elasticsearch and kibana service ports", func() { + out := extensions.ApplyModifiers(render.GuardianName, ctxWith(render.GuardianExtensionContext{}), newObjs()) + svc, _ := extensions.FindObject[*corev1.Service](out, render.GuardianServiceName) + names := []string{} + for _, p := range svc.Spec.Ports { + names = append(names, p.Name) + } + Expect(names).To(ContainElements("https", "elasticsearch", "kibana")) + }) + + It("replaces the cluster role rules and adds impersonation", func() { + gc := render.GuardianExtensionContext{ + Impersonation: &operatorv1.Impersonation{Users: []string{"foo"}, Groups: []string{"bar"}}, + } + out := extensions.ApplyModifiers(render.GuardianName, ctxWith(gc), newObjs()) + role, _ := extensions.FindObject[*rbacv1.ClusterRole](out, render.GuardianClusterRoleName) + + // The single OSS placeholder rule is gone, replaced by the enterprise set. + Expect(role.Rules).NotTo(ContainElement(rbacv1.PolicyRule{Verbs: []string{"get"}})) + Expect(role.Rules).To(ContainElement(HaveField("ResourceNames", Equal([]string{"foo"})))) + Expect(role.Rules).To(ContainElement(HaveField("ResourceNames", Equal([]string{"bar"})))) + }) + + It("adds the CA bundle env to the guardian container", func() { + gc := render.GuardianExtensionContext{TrustedBundleMountPath: "/ca/bundle"} + out := extensions.ApplyModifiers(render.GuardianName, ctxWith(gc), newObjs()) + dep, _ := extensions.FindObject[*appsv1.Deployment](out, render.GuardianDeploymentName) + Expect(dep.Spec.Template.Spec.Containers[0].Env).To(ContainElement(corev1.EnvVar{Name: "GUARDIAN_PROMETHEUS_CA_BUNDLE_PATH", Value: "/ca/bundle"})) + }) + + It("does nothing for the Calico variant", func() { + ctx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}} + out := extensions.ApplyModifiers(render.GuardianName, ctx, newObjs()) + Expect(out).To(HaveLen(len(newObjs()))) + role, _ := extensions.FindObject[*rbacv1.ClusterRole](out, render.GuardianClusterRoleName) + Expect(role.Rules).To(Equal([]rbacv1.PolicyRule{{Verbs: []string{"get"}}})) + }) +}) diff --git a/pkg/enterprise/register.go b/pkg/enterprise/register.go index 761f1736b7..20589ad0fa 100644 --- a/pkg/enterprise/register.go +++ b/pkg/enterprise/register.go @@ -21,5 +21,6 @@ func Register() { registerTypha() registerNode() registerWindows() + registerGuardian() registerInstallation() } diff --git a/pkg/render/guardian.go b/pkg/render/guardian.go index 8fd24d35b6..8c1f30cf3c 100644 --- a/pkg/render/guardian.go +++ b/pkg/render/guardian.go @@ -45,7 +45,6 @@ import ( "github.com/tigera/operator/pkg/render/common/networkpolicy" "github.com/tigera/operator/pkg/render/common/secret" "github.com/tigera/operator/pkg/render/common/securitycontext" - "github.com/tigera/operator/pkg/render/common/securitycontextconstraints" "github.com/tigera/operator/pkg/tls/certificatemanagement" ) @@ -150,32 +149,40 @@ func (c *GuardianComponent) SupportedOSType() rmeta.OSType { return rmeta.OSTypeLinux } +func (c *GuardianComponent) ModifierKey() string { return GuardianName } + +// GuardianExtensionContext is the per-component context the guardian modifier +// reads (via RenderContext.Component). It carries the inputs the enterprise +// guardian behavior needs that a modifier can't derive from the installation: +// the management cluster's impersonation config, whether we're on OpenShift, +// and the trusted bundle mount path the CA env vars reference. +type GuardianExtensionContext struct { + OpenShift bool + Impersonation *operatorv1.Impersonation + TrustedBundleMountPath string +} + +func (c *GuardianComponent) ExtensionContext() any { + var impersonation *operatorv1.Impersonation + if c.cfg.ManagementClusterConnection != nil { + impersonation = c.cfg.ManagementClusterConnection.Spec.Impersonation + } + return GuardianExtensionContext{ + OpenShift: c.cfg.OpenShift, + Impersonation: impersonation, + TrustedBundleMountPath: c.cfg.TrustedCertBundle.MountPath(), + } +} + func (c *GuardianComponent) Objects() ([]client.Object, []client.Object) { objs := []client.Object{ - // common RBAC for EE and OSS c.serviceAccount(), c.clusterRole(), c.clusterRoleBinding(), - } - - if c.cfg.Installation.Variant.IsEnterprise() { - // Enterprise-specific RBAC and settings - objs = append(objs, - c.secretsRole(), - c.secretRoleBinding(), - // Install default UI settings for this managed cluster. - managerClusterWideSettingsGroup(), - managerUserSpecificSettingsGroup(), - managerClusterWideTigeraLayer(), - managerClusterWideDefaultView(), - ) - } - - objs = append(objs, c.deployment(), c.service(), secret.CopyToNamespace(GuardianNamespace, c.cfg.TunnelSecret)[0], - ) + } return objs, deprecatedObjects() } @@ -197,28 +204,6 @@ func (c *GuardianComponent) service() *corev1.Service { }, } - if c.cfg.Installation.Variant.IsEnterprise() { - ports = append(ports, - corev1.ServicePort{ - Name: "elasticsearch", - Port: 9200, - TargetPort: intstr.IntOrString{ - Type: intstr.Int, - IntVal: 8080, - }, - Protocol: corev1.ProtocolTCP, - }, - corev1.ServicePort{ - Name: "kibana", - Port: 5601, - TargetPort: intstr.IntOrString{ - Type: intstr.Int, - IntVal: 8080, - }, - Protocol: corev1.ProtocolTCP, - }, - ) - } return &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: GuardianServiceName, @@ -241,87 +226,42 @@ func (c *GuardianComponent) serviceAccount() *corev1.ServiceAccount { } func (c *GuardianComponent) clusterRole() *rbacv1.ClusterRole { - var policyRules []rbacv1.PolicyRule - if c.cfg.Installation.Variant.IsEnterprise() { - impersonation := c.cfg.ManagementClusterConnection.Spec.Impersonation - if impersonation != nil { - if impersonation.Users != nil { - policyRules = append(policyRules, - rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"users"}, - ResourceNames: impersonation.Users, - Verbs: []string{"impersonate"}, - }) - } - if impersonation.Groups != nil { - policyRules = append(policyRules, - rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"groups"}, - ResourceNames: impersonation.Groups, - Verbs: []string{"impersonate"}, - }) - } - if impersonation.ServiceAccounts != nil { - policyRules = append(policyRules, - rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"serviceaccounts"}, - ResourceNames: impersonation.ServiceAccounts, - Verbs: []string{"impersonate"}, - }) - } - } - - policyRules = append(policyRules, rulesForManagementClusterRequests(c.cfg.OpenShift)...) - - if c.cfg.OpenShift { - policyRules = append(policyRules, rbacv1.PolicyRule{ - APIGroups: []string{"security.openshift.io"}, - Resources: []string{"securitycontextconstraints"}, - Verbs: []string{"use"}, - ResourceNames: []string{securitycontextconstraints.NonRootV2}, - }) - } - } else { - policyRules = append(policyRules, - rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"namespaces", "services", "pods"}, - Verbs: []string{"get", "list", "watch"}, - }, - rbacv1.PolicyRule{ - APIGroups: []string{"apps"}, - Resources: []string{"deployments", "replicasets", "statefulsets", "daemonsets"}, - Verbs: []string{"get", "list", "watch"}, - }, - rbacv1.PolicyRule{ - APIGroups: []string{"networking.k8s.io"}, - Resources: []string{"networkpolicies"}, - Verbs: []string{"get", "list", "watch"}, - }, - rbacv1.PolicyRule{ - APIGroups: []string{"projectcalico.org"}, - Resources: []string{ - "clusterinformations", - "tiers", - "stagednetworkpolicies", - "tier.stagednetworkpolicies", - "stagedglobalnetworkpolicies", - "tier.stagedglobalnetworkpolicies", - "stagedkubernetesnetworkpolicies", - "tier.stagedkubernetesnetworkpolicies", - "networkpolicies", - "tier.networkpolicies", - "globalnetworkpolicies", - "tier.globalnetworkpolicies", - "globalnetworksets", - "networksets", - }, - Verbs: []string{"get", "list", "watch"}, + policyRules := []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{"namespaces", "services", "pods"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + APIGroups: []string{"apps"}, + Resources: []string{"deployments", "replicasets", "statefulsets", "daemonsets"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + APIGroups: []string{"networking.k8s.io"}, + Resources: []string{"networkpolicies"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{ + "clusterinformations", + "tiers", + "stagednetworkpolicies", + "tier.stagednetworkpolicies", + "stagedglobalnetworkpolicies", + "tier.stagedglobalnetworkpolicies", + "stagedkubernetesnetworkpolicies", + "tier.stagedkubernetesnetworkpolicies", + "networkpolicies", + "tier.networkpolicies", + "globalnetworkpolicies", + "tier.globalnetworkpolicies", + "globalnetworksets", + "networksets", }, - ) + Verbs: []string{"get", "list", "watch"}, + }, } return &rbacv1.ClusterRole{ @@ -354,47 +294,6 @@ func (c *GuardianComponent) clusterRoleBinding() *rbacv1.ClusterRoleBinding { } } -// secretRole creates a Role that allows the management cluster to provision secrets to the tigera-operator Namespace. -// This is used to push secrets used by the managed cluster to access / authenticate with the management cluster. -func (c *GuardianComponent) secretsRole() *rbacv1.Role { - return &rbacv1.Role{ - TypeMeta: metav1.TypeMeta{Kind: "Role", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: GuardianSecretsRole, - Namespace: common.OperatorNamespace(), - }, - Rules: []rbacv1.PolicyRule{ - { - APIGroups: []string{""}, - Resources: []string{"secrets"}, - Verbs: []string{"create", "delete", "deletecollection", "update"}, - }, - }, - } -} - -func (c *GuardianComponent) secretRoleBinding() *rbacv1.RoleBinding { - return &rbacv1.RoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: GuardianSecretsRoleBindingName, - Namespace: common.OperatorNamespace(), - }, - RoleRef: rbacv1.RoleRef{ - APIGroup: "rbac.authorization.k8s.io", - Kind: "Role", - Name: GuardianSecretsRole, - }, - Subjects: []rbacv1.Subject{ - { - Kind: "ServiceAccount", - Name: GuardianServiceAccountName, - Namespace: GuardianNamespace, - }, - }, - } -} - func (c *GuardianComponent) deployment() *appsv1.Deployment { var replicas int32 = 1 @@ -468,14 +367,6 @@ func (c *GuardianComponent) container() []corev1.Container { } envVars = append(envVars, c.cfg.Installation.Proxy.EnvVars()...) - if c.cfg.Installation.Variant.IsEnterprise() { - envVars = append(envVars, - corev1.EnvVar{Name: "GUARDIAN_PACKET_CAPTURE_CA_BUNDLE_PATH", Value: c.cfg.TrustedCertBundle.MountPath()}, - corev1.EnvVar{Name: "GUARDIAN_PROMETHEUS_CA_BUNDLE_PATH", Value: c.cfg.TrustedCertBundle.MountPath()}, - corev1.EnvVar{Name: "GUARDIAN_QUERYSERVER_CA_BUNDLE_PATH", Value: c.cfg.TrustedCertBundle.MountPath()}, - ) - } - if c.cfg.GuardianClientKeyPair != nil { envVars = append(envVars, corev1.EnvVar{ @@ -782,304 +673,6 @@ func GuardianService(clusterDomain string) string { return fmt.Sprintf("https://%s.%s.svc.%s:%d", GuardianServiceName, GuardianNamespace, clusterDomain, 443) } -// rulesForManagementClusterRequests returns the set of RBAC rules needed by Guardian in order to -// satisfy requests from the management cluster over the tunnel. -func rulesForManagementClusterRequests(isOpenShift bool) []rbacv1.PolicyRule { - rules := []rbacv1.PolicyRule{ - // Common rules required to handle requests from multiple components in the management cluster. - { - // ID uses read-only permissions and kube-controllers uses both read and write verbs. - APIGroups: []string{""}, - Resources: []string{"configmaps"}, - Verbs: []string{"create", "delete", "get", "list", "update", "watch"}, - }, - { - // Allows Linseed to watch namespaces before copying its token. - // Also enables PolicyRecommendation to watch namespaces, - // and Manager/kube-controllers to list them. - APIGroups: []string{""}, - Resources: []string{"namespaces"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - // kube-controllers watches Nodes to monitor for deletions. - // Manager performs a list operation on Nodes. - APIGroups: []string{""}, - Resources: []string{"nodes"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - // kube-controllers watches Pods to verify existence for IPAM garbage collection. - // Manager performs get operations on Pods. - APIGroups: []string{""}, - Resources: []string{"pods"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - // The Federated Services Controller needs access to the remote kubeconfig secret - // in order to create a remote syncer. - APIGroups: []string{""}, - Resources: []string{"secrets"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - // Manager uses list; kube-controllers uses 'get', 'list', 'watch', 'update'. - APIGroups: []string{""}, - Resources: []string{"services"}, - Verbs: []string{"get", "list", "update", "watch"}, - }, - { - // Needed by kube-controllers to validate licenses; also used by ID. - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"licensekeys"}, - Verbs: []string{"get", "watch"}, - }, - { - // Manager uses list; PolicyRecommendation & ID uses all verbs. - APIGroups: []string{"projectcalico.org"}, - Resources: []string{ - "globalnetworksets", - "networkpolicies", - "tier.networkpolicies", - "stagednetworkpolicies", - "tier.stagednetworkpolicies", - }, - Verbs: []string{"create", "delete", "get", "list", "patch", "update", "watch"}, - }, - { - // Manager uses list; PolicyRecommendation uses all verbs. - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"tiers"}, - Verbs: []string{"create", "delete", "get", "list", "patch", "update", "watch"}, - }, - // Rules needed by guardian to handle manager authorization reviews. - { - APIGroups: []string{"rbac.authorization.k8s.io"}, - Resources: []string{"clusterroles", "clusterrolebindings", "roles", "rolebindings"}, - Verbs: []string{"list", "get"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"uisettings", "uisettingsgroups"}, - Verbs: []string{"list", "get"}, - }, - - // Rules needed by guardian to handle other manager requests. - { - APIGroups: []string{""}, - Resources: []string{"events"}, - Verbs: []string{"list"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"serviceaccounts"}, - Verbs: []string{"list"}, - }, - { - // Allow query server talk to Prometheus via the manager user. - APIGroups: []string{""}, - Resources: []string{"services/proxy"}, - ResourceNames: []string{ - "calico-node-prometheus:9090", - "https:calico-api:8080", - }, - Verbs: []string{"create", "get"}, - }, - { - APIGroups: []string{"apps"}, - Resources: []string{"daemonsets", "replicasets", "statefulsets"}, - Verbs: []string{"list"}, - }, - { - APIGroups: []string{"authentication.k8s.io"}, - Resources: []string{"tokenreviews"}, - Verbs: []string{"create"}, - }, - { - APIGroups: []string{"authorization.k8s.io"}, - Resources: []string{"subjectaccessreviews"}, - Verbs: []string{"create"}, - }, - { - APIGroups: []string{"networking.k8s.io"}, - Resources: []string{"networkpolicies"}, - Verbs: []string{"get", "list"}, - }, - { - APIGroups: []string{"policy.networking.k8s.io"}, - Resources: []string{ - "clusternetworkpolicies", - "adminnetworkpolicies", - "baselineadminnetworkpolicies", - }, - Verbs: []string{"list"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"alertexceptions"}, - Verbs: []string{"get", "list", "update"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"felixconfigurations"}, - ResourceNames: []string{"default"}, - Verbs: []string{"get"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{ - "globalnetworkpolicies", - "networksets", - "stagedglobalnetworkpolicies", - "stagedkubernetesnetworkpolicies", - "tier.globalnetworkpolicies", - "tier.stagedglobalnetworkpolicies", - }, - Verbs: []string{"list"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"hostendpoints"}, - Verbs: []string{"list"}, - }, - - // Rules needed by guardian to handle policy recommendation requests. - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{ - "policyrecommendationscopes", - "policyrecommendationscopes/status", - }, - Verbs: []string{"create", "delete", "get", "list", "patch", "update", "watch"}, - }, - - // Rules needed by guardian to handle calico-kube-controller requests. - { - // Nodes are watched to monitor for deletions. - APIGroups: []string{""}, - Resources: []string{"endpoints"}, - Verbs: []string{"create", "delete", "get", "list", "update", "watch"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"services/status"}, - Verbs: []string{"get", "list", "update", "watch"}, - }, - { - // Needs to manage hostendpoints. - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"hostendpoints"}, - Verbs: []string{"create", "delete", "get", "list", "update", "watch"}, - }, - { - // Needs access to update clusterinformations. - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"clusterinformations"}, - Verbs: []string{"create", "get", "list", "update", "watch"}, - }, - { - // Needs to manipulate kubecontrollersconfiguration, which contains its config. - // It creates a default if none exists, and updates status as well. - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"kubecontrollersconfigurations"}, - Verbs: []string{"create", "get", "list", "update", "watch"}, - }, - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"tiers"}, - Verbs: []string{"create"}, - }, - { - APIGroups: []string{"crd.projectcalico.org", "projectcalico.org"}, - Resources: []string{"deeppacketinspections"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"deeppacketinspections/status"}, - Verbs: []string{"update"}, - }, - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"packetcaptures"}, - Verbs: []string{"get", "list", "update"}, - }, - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"remoteclusterconfigurations"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"licensekeys"}, - Verbs: []string{"create", "get", "list", "update", "watch"}, - }, - { - // Grant permissions to access ClusterInformation resources in managed clusters. - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"clusterinformations"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - APIGroups: []string{"usage.tigera.io"}, - Resources: []string{"licenseusagereports"}, - Verbs: []string{"create", "delete", "get", "list", "update", "watch"}, - }, - - // Rules needed by guardian to handle Intrusion detection requests. - { - APIGroups: []string{""}, - Resources: []string{"podtemplates"}, - Verbs: []string{"get"}, - }, - { - APIGroups: []string{"apps"}, - Resources: []string{"deployments"}, - Verbs: []string{"get"}, - }, - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"alertexceptions"}, - Verbs: []string{"get", "list"}, - }, - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"securityeventwebhooks"}, - Verbs: []string{"get", "list", "update", "watch"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{ - "globalalerts", - "globalalerts/status", - "globalthreatfeeds", - "globalthreatfeeds/status", - }, - Verbs: []string{"create", "delete", "get", "list", "patch", "update", "watch"}, - }, - // Rules needed to fetch the compliance reports - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"globalreporttypes", "globalreports"}, - Verbs: []string{"get", "list", "watch"}, - }, - } - - // Rules needed by policy recommendation in openshift. - if isOpenShift { - rules = append(rules, - rbacv1.PolicyRule{ - APIGroups: []string{"security.openshift.io"}, - Resources: []string{"securitycontextconstraints"}, - Verbs: []string{"use"}, - ResourceNames: []string{securitycontextconstraints.HostNetworkV2}, - }, - ) - } - - return rules -} - func deprecatedObjects() []client.Object { return []client.Object{ // All the Guardian objects were moved to "calico-system" circa Calico v3.30, and so the legacy tigera-guardian diff --git a/pkg/render/guardian_test.go b/pkg/render/guardian_test.go index 7e82472a41..7d2b454abb 100644 --- a/pkg/render/guardian_test.go +++ b/pkg/render/guardian_test.go @@ -36,6 +36,7 @@ import ( "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/controller/certificatemanager" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" rmeta "github.com/tigera/operator/pkg/render/common/meta" "github.com/tigera/operator/pkg/render/common/networkpolicy" @@ -43,6 +44,19 @@ import ( "github.com/tigera/operator/pkg/render/testutils" ) +// guardianObjects renders the guardian component and applies the registered +// enterprise modifier the way the componentHandler does. +func guardianObjects(cfg *render.GuardianConfiguration) []client.Object { + g := render.Guardian(cfg) + ExpectWithOffset(1, g.ResolveImages(nil)).To(BeNil()) + objs, _ := g.Objects() + rc := extensions.RenderContext{Installation: cfg.Installation} + if p, ok := g.(render.ExtensionContextProvider); ok { + rc.Component = p.ExtensionContext() + } + return extensions.ApplyModifiers(render.GuardianName, rc, objs) +} + var _ = Describe("Rendering tests", func() { var cfg *render.GuardianConfiguration var g render.Component @@ -95,6 +109,13 @@ var _ = Describe("Rendering tests", func() { g = render.Guardian(cfg) Expect(g.ResolveImages(nil)).To(BeNil()) resources, deleteResources = g.Objects() + // Apply the registered enterprise modifier the way the componentHandler + // does, so these enterprise tests exercise the integrated output. + rc := extensions.RenderContext{Installation: cfg.Installation} + if p, ok := g.(render.ExtensionContextProvider); ok { + rc.Component = p.ExtensionContext() + } + resources = extensions.ApplyModifiers(render.GuardianName, rc, resources) } BeforeEach(func() { @@ -189,8 +210,7 @@ var _ = Describe("Rendering tests", func() { }, } - g := render.Guardian(cfg) - resources, _ := g.Objects() + resources := guardianObjects(cfg) Expect(resources).ToNot(BeNil()) clusterRole, ok := rtest.GetResource(resources, render.GuardianClusterRoleName, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) @@ -230,8 +250,7 @@ var _ = Describe("Rendering tests", func() { }, } - g := render.Guardian(cfg) - resources, _ := g.Objects() + resources := guardianObjects(cfg) Expect(resources).ToNot(BeNil()) clusterRole, ok := rtest.GetResource(resources, render.GuardianClusterRoleName, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) @@ -270,8 +289,7 @@ var _ = Describe("Rendering tests", func() { }, } - g := render.Guardian(cfg) - resources, _ := g.Objects() + resources := guardianObjects(cfg) Expect(resources).ToNot(BeNil()) clusterRole, ok := rtest.GetResource(resources, render.GuardianClusterRoleName, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) @@ -301,9 +319,7 @@ var _ = Describe("Rendering tests", func() { It("should render SecurityContextConstrains properly when provider is OpenShift", func() { cfg.Installation.KubernetesProvider = operatorv1.ProviderOpenShift cfg.OpenShift = true - component := render.Guardian(cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources := guardianObjects(cfg) role := rtest.GetResource(resources, render.GuardianClusterRoleName, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) Expect(role.Rules).To(ContainElement(rbacv1.PolicyRule{ @@ -441,8 +457,7 @@ var _ = Describe("guardian", func() { } }) It("should render when disabled", func() { - g := render.Guardian(cfg) - resources, _ := g.Objects() + resources := guardianObjects(cfg) Expect(resources).ToNot(BeNil()) deployment := rtest.GetResource(resources, render.GuardianDeploymentName, render.GuardianNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) @@ -452,8 +467,7 @@ var _ = Describe("guardian", func() { It("should render when set to disabled", func() { cfg.TunnelCAType = operatorv1.CATypeTigera - g := render.Guardian(cfg) - resources, _ := g.Objects() + resources := guardianObjects(cfg) Expect(resources).ToNot(BeNil()) deployment := rtest.GetResource(resources, render.GuardianDeploymentName, render.GuardianNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) @@ -464,8 +478,7 @@ var _ = Describe("guardian", func() { It("should render when enabled", func() { cfg.TunnelCAType = operatorv1.CATypePublic - g := render.Guardian(cfg) - resources, _ := g.Objects() + resources := guardianObjects(cfg) Expect(resources).ToNot(BeNil()) deployment := rtest.GetResource(resources, render.GuardianDeploymentName, render.GuardianNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) @@ -503,8 +516,7 @@ var _ = Describe("guardian", func() { }, } - g := render.Guardian(cfg) - resources, _ := g.Objects() + resources := guardianObjects(cfg) Expect(resources).ToNot(BeNil()) deployment, ok := rtest.GetResource(resources, render.GuardianDeploymentName, render.GuardianNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) diff --git a/pkg/render/manager.go b/pkg/render/manager.go index eaa20519db..d08937a686 100644 --- a/pkg/render/manager.go +++ b/pkg/render/manager.go @@ -262,10 +262,10 @@ func (c *managerComponent) Objects() ([]client.Object, []client.Object) { // For multi-tenant environments, the management cluster itself isn't shown in the UI so we only need to create these // when there is no tenant. objsToCreate = append(objsToCreate, - managerClusterWideSettingsGroup(), - managerUserSpecificSettingsGroup(), - managerClusterWideTigeraLayer(), - managerClusterWideDefaultView(), + ManagerClusterWideSettingsGroup(), + ManagerUserSpecificSettingsGroup(), + ManagerClusterWideTigeraLayer(), + ManagerClusterWideDefaultView(), ) // Continue to create the legacy namespace so that we can create our external name service that points to the new // manager service. This will help ease transition for customers and avoid outages caused by the name and namespace @@ -1337,10 +1337,10 @@ func (c *managerComponent) multiTenantManagedClustersAccess() []client.Object { return objects } -// managerClusterWideSettingsGroup returns a UISettingsGroup with the description "cluster-wide settings" +// ManagerClusterWideSettingsGroup returns a UISettingsGroup with the description "cluster-wide settings" // // Calico Enterprise only -func managerClusterWideSettingsGroup() *v3.UISettingsGroup { +func ManagerClusterWideSettingsGroup() *v3.UISettingsGroup { return &v3.UISettingsGroup{ TypeMeta: metav1.TypeMeta{Kind: "UISettingsGroup", APIVersion: "projectcalico.org/v3"}, ObjectMeta: metav1.ObjectMeta{ @@ -1352,10 +1352,10 @@ func managerClusterWideSettingsGroup() *v3.UISettingsGroup { } } -// managerUserSpecificSettingsGroup returns a UISettingsGroup with the description "user settings" +// ManagerUserSpecificSettingsGroup returns a UISettingsGroup with the description "user settings" // // Calico Enterprise only -func managerUserSpecificSettingsGroup() *v3.UISettingsGroup { +func ManagerUserSpecificSettingsGroup() *v3.UISettingsGroup { return &v3.UISettingsGroup{ TypeMeta: metav1.TypeMeta{Kind: "UISettingsGroup", APIVersion: "projectcalico.org/v3"}, ObjectMeta: metav1.ObjectMeta{ @@ -1368,11 +1368,11 @@ func managerUserSpecificSettingsGroup() *v3.UISettingsGroup { } } -// managerClusterWideTigeraLayer returns a UISettings layer belonging to the cluster-wide settings group that contains +// ManagerClusterWideTigeraLayer returns a UISettings layer belonging to the cluster-wide settings group that contains // all of the tigera namespaces. // // Calico Enterprise only -func managerClusterWideTigeraLayer() *v3.UISettings { +func ManagerClusterWideTigeraLayer() *v3.UISettings { namespaces := []string{ "tigera-compliance", "tigera-dex", @@ -1418,11 +1418,11 @@ func managerClusterWideTigeraLayer() *v3.UISettings { } } -// managerClusterWideDefaultView returns a UISettings view belonging to the cluster-wide settings group that shows +// ManagerClusterWideDefaultView returns a UISettings view belonging to the cluster-wide settings group that shows // everything and uses the tigera-infrastructure layer. // // Calico Enterprise only -func managerClusterWideDefaultView() *v3.UISettings { +func ManagerClusterWideDefaultView() *v3.UISettings { return &v3.UISettings{ TypeMeta: metav1.TypeMeta{Kind: "UISettings", APIVersion: "projectcalico.org/v3"}, ObjectMeta: metav1.ObjectMeta{ From 0351e221f5511b00537320cdc7e8662d4b0bbd21 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Tue, 9 Jun 2026 15:35:14 -0700 Subject: [PATCH 23/38] De-variant the guardian network policy Move the enterprise guardian network policy out of core render. GuardianPolicy is now an extensible component that renders the OSS policy; the enterprise modifier replaces it with the management-cluster policy, built entirely from the component's ExtensionContext (URL, pod proxies, OpenShift, egress-policy flag). Building the egress rules can fail on proxy URL parsing - the modifier drops the policy on failure, matching the old behavior of omitting it rather than installing a partial one. guardian.go now carries no IsEnterprise branches. --- pkg/enterprise/guardian.go | 172 ++++++++++++++++++++++++ pkg/render/guardian.go | 256 ++++++------------------------------ pkg/render/guardian_test.go | 10 +- 3 files changed, 222 insertions(+), 216 deletions(-) diff --git a/pkg/enterprise/guardian.go b/pkg/enterprise/guardian.go index bcac8a48e1..c85bad2680 100644 --- a/pkg/enterprise/guardian.go +++ b/pkg/enterprise/guardian.go @@ -15,6 +15,10 @@ package enterprise import ( + "net" + "net/url" + + "github.com/sirupsen/logrus" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -22,17 +26,185 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" "sigs.k8s.io/controller-runtime/pkg/client" + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + "github.com/tigera/api/pkg/lib/numorstring" + operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/render/common/networkpolicy" "github.com/tigera/operator/pkg/render/common/securitycontextconstraints" + operatorurl "github.com/tigera/operator/pkg/url" ) func registerGuardian() { extensions.Register(operatorv1.CalicoEnterprise, render.GuardianName, extensions.Extension{ Modify: modifyGuardian, }) + extensions.Register(operatorv1.CalicoEnterprise, render.ComponentNameGuardianPolicy, extensions.Extension{ + Modify: modifyGuardianPolicy, + }) +} + +// modifyGuardianPolicy replaces the core OSS guardian network policy with the +// enterprise management-cluster policy. Building the enterprise egress rules can +// fail (proxy URL parsing); on failure we drop the policy entirely, matching the +// core behavior of omitting it rather than installing a partial policy. +func modifyGuardianPolicy(ctx extensions.RenderContext, objs []client.Object) []client.Object { + gpc, _ := ctx.Component.(render.GuardianPolicyExtensionContext) + + policy, ok := extensions.FindObject[*v3.NetworkPolicy](objs, render.GuardianPolicyName) + if !ok { + return objs + } + + spec, err := enterpriseGuardianPolicySpec(gpc) + if err != nil { + logrus.WithError(err).Error("Failed to build guardian network policy, policy will be omitted") + return removeObject(objs, policy) + } + policy.Spec = spec + return objs +} + +func removeObject(objs []client.Object, drop client.Object) []client.Object { + out := objs[:0] + for _, o := range objs { + if o != drop { + out = append(out, o) + } + } + return out +} + +// enterpriseGuardianPolicySpec builds the network policy spec for guardian in a +// managed cluster: egress to the management cluster components and the tunnel +// destination(s), and ingress from the management-cluster components that reach +// back over the tunnel. +func enterpriseGuardianPolicySpec(gpc render.GuardianPolicyExtensionContext) (v3.NetworkPolicySpec, error) { + egressRules := []v3.Rule{ + { + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: render.PacketCaptureEntityRule, + }, + } + egressRules = networkpolicy.AppendDNSEgressRules(egressRules, gpc.OpenShift) + egressRules = append(egressRules, []v3.Rule{ + { + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: networkpolicy.KubeAPIServerEntityRule, + }, + { + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: networkpolicy.PrometheusEntityRule, + }, + { + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: render.TigeraAPIServerEntityRule, + }, + }...) + + // Create an egress rule for each unique destination the guardian pods connect + // to. With multiple pods whose proxy settings differ, there are multiple + // destinations that must be allowed. + allowedDestinations := map[string]bool{} + for _, podProxyConfig := range render.ProcessPodProxies(gpc.PodProxies) { + var proxyURL *url.URL + var err error + if podProxyConfig != nil && podProxyConfig.HTTPSProxy != "" { + // The scheme is HTTPS, as we establish an mTLS session with the target. + // We expect the URL to be of the form host:port. + targetURL := &url.URL{Scheme: "https", Host: gpc.URL} + proxyURL, err = podProxyConfig.ProxyFunc()(targetURL) + if err != nil { + return v3.NetworkPolicySpec{}, err + } + } + + var tunnelDestinationHostPort string + if proxyURL != nil { + proxyHostPort, err := operatorurl.ParseHostPortFromHTTPProxyURL(proxyURL) + if err != nil { + return v3.NetworkPolicySpec{}, err + } + tunnelDestinationHostPort = proxyHostPort + } else { + // gpc.URL has host:port form. + tunnelDestinationHostPort = gpc.URL + } + + if allowedDestinations[tunnelDestinationHostPort] { + continue + } + + host, port, err := net.SplitHostPort(tunnelDestinationHostPort) + if err != nil { + return v3.NetworkPolicySpec{}, err + } + parsedPort, err := numorstring.PortFromString(port) + if err != nil { + return v3.NetworkPolicySpec{}, err + } + parsedIP := net.ParseIP(host) + if parsedIP == nil { + // Domain-based egress rules require the EgressAccessControl license feature. + if !gpc.IncludeEgressNetworkPolicy { + continue + } + egressRules = append(egressRules, v3.Rule{ + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: v3.EntityRule{ + Domains: []string{host}, + Ports: []numorstring.Port{parsedPort}, + }, + }) + allowedDestinations[tunnelDestinationHostPort] = true + } else { + netSuffix := "/32" + if parsedIP.To4() == nil { + netSuffix = "/128" + } + egressRules = append(egressRules, v3.Rule{ + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: v3.EntityRule{ + Nets: []string{parsedIP.String() + netSuffix}, + Ports: []numorstring.Port{parsedPort}, + }, + }) + allowedDestinations[tunnelDestinationHostPort] = true + } + } + + egressRules = append(egressRules, v3.Rule{Action: v3.Pass}) + + dest := v3.EntityRule{Ports: networkpolicy.Ports(render.GuardianTargetPort)} + helper := networkpolicy.DefaultHelper() + ingressRules := []v3.Rule{ + {Action: v3.Allow, Protocol: &networkpolicy.TCPProtocol, Source: render.FluentdSourceEntityRule, Destination: dest}, + {Action: v3.Allow, Protocol: &networkpolicy.TCPProtocol, Source: helper.ComplianceBenchmarkerSourceEntityRule(), Destination: dest}, + {Action: v3.Allow, Protocol: &networkpolicy.TCPProtocol, Source: helper.ComplianceReporterSourceEntityRule(), Destination: dest}, + {Action: v3.Allow, Protocol: &networkpolicy.TCPProtocol, Source: helper.ComplianceSnapshotterSourceEntityRule(), Destination: dest}, + {Action: v3.Allow, Protocol: &networkpolicy.TCPProtocol, Source: helper.ComplianceControllerSourceEntityRule(), Destination: dest}, + {Action: v3.Allow, Protocol: &networkpolicy.TCPProtocol, Source: render.IntrusionDetectionSourceEntityRule, Destination: dest}, + {Action: v3.Allow, Protocol: &networkpolicy.TCPProtocol, Source: render.IntrusionDetectionInstallerSourceEntityRule, Destination: dest}, + {Action: v3.Allow, Protocol: &networkpolicy.TCPProtocol, Destination: dest}, + } + + return v3.NetworkPolicySpec{ + Order: &networkpolicy.HighPrecedenceOrder, + Tier: networkpolicy.CalicoTierName, + Selector: networkpolicy.KubernetesAppSelector(render.GuardianName), + Types: []v3.PolicyType{v3.PolicyTypeIngress, v3.PolicyTypeEgress}, + Ingress: ingressRules, + Egress: egressRules, + }, nil } // modifyGuardian layers Calico Enterprise behavior onto the rendered guardian diff --git a/pkg/render/guardian.go b/pkg/render/guardian.go index 8c1f30cf3c..f723469447 100644 --- a/pkg/render/guardian.go +++ b/pkg/render/guardian.go @@ -18,13 +18,9 @@ package render import ( "fmt" - "net" - "net/url" "golang.org/x/net/http/httpproxy" - operatorurl "github.com/tigera/operator/pkg/url" - appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" netv1 "k8s.io/api/networking/v1" @@ -35,8 +31,6 @@ import ( v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" - "github.com/tigera/api/pkg/lib/numorstring" - operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/components" @@ -84,27 +78,51 @@ func Guardian(cfg *GuardianConfiguration) Component { } } +// GuardianPolicy renders the guardian network policy. The core operator renders +// the OSS policy; the enterprise modifier (keyed ComponentNameGuardianPolicy) +// replaces it with the management-cluster policy. The error return is retained +// for callers but is always nil now that the fallible enterprise computation +// lives in the modifier. func GuardianPolicy(cfg *GuardianConfiguration) (Component, error) { - var policies []client.Object + return &guardianPolicyComponent{cfg: cfg}, nil +} - guardianAccessPolicy, err := guardianCalicoSystemPolicy(cfg) - if err != nil { - return nil, err - } - if guardianAccessPolicy != nil { - policies = []client.Object{ - guardianAccessPolicy, - } +const ComponentNameGuardianPolicy = "guardian-policy" + +type guardianPolicyComponent struct { + cfg *GuardianConfiguration +} + +func (c *guardianPolicyComponent) ResolveImages(*operatorv1.ImageSet) error { return nil } +func (c *guardianPolicyComponent) SupportedOSType() rmeta.OSType { return rmeta.OSTypeAny } +func (c *guardianPolicyComponent) Ready() bool { return true } +func (c *guardianPolicyComponent) ModifierKey() string { return ComponentNameGuardianPolicy } + +// GuardianPolicyExtensionContext is the per-component context the guardian +// policy modifier reads (via RenderContext.Component). The enterprise guardian +// network policy is built entirely from these inputs. +type GuardianPolicyExtensionContext struct { + URL string + PodProxies []*httpproxy.Config + OpenShift bool + IncludeEgressNetworkPolicy bool +} + +func (c *guardianPolicyComponent) ExtensionContext() any { + return GuardianPolicyExtensionContext{ + URL: c.cfg.URL, + PodProxies: c.cfg.PodProxies, + OpenShift: c.cfg.OpenShift, + IncludeEgressNetworkPolicy: c.cfg.IncludeEgressNetworkPolicy, } +} - return NewPassthrough( - policies, - []client.Object{ - // allow-tigera Tier was renamed to calico-system - networkpolicy.DeprecatedAllowTigeraNetworkPolicyObject("guardian-access", GuardianNamespace), - networkpolicy.DeprecatedAllowTigeraNetworkPolicyObject("default-deny", GuardianNamespace), - }, - ), nil +func (c *guardianPolicyComponent) Objects() ([]client.Object, []client.Object) { + return []client.Object{ossNetworkPolicy()}, []client.Object{ + // allow-tigera Tier was renamed to calico-system + networkpolicy.DeprecatedAllowTigeraNetworkPolicyObject("guardian-access", GuardianNamespace), + networkpolicy.DeprecatedAllowTigeraNetworkPolicyObject("default-deny", GuardianNamespace), + } } // GuardianConfiguration contains all the config information needed to render the component. @@ -467,198 +485,6 @@ func ossNetworkPolicy() *v3.NetworkPolicy { } } -func guardianCalicoSystemPolicy(cfg *GuardianConfiguration) (*v3.NetworkPolicy, error) { - if !cfg.Installation.Variant.IsEnterprise() { - return ossNetworkPolicy(), nil - } - - egressRules := []v3.Rule{ - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: PacketCaptureEntityRule, - }, - } - egressRules = networkpolicy.AppendDNSEgressRules(egressRules, cfg.OpenShift) - egressRules = append(egressRules, []v3.Rule{ - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: networkpolicy.KubeAPIServerEntityRule, - }, - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: networkpolicy.PrometheusEntityRule, - }, - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: TigeraAPIServerEntityRule, - }, - }...) - - // The loop below creates an egress rule for each unique destination that the Guardian pods connect to. If there are - // multiple guardian pods and their proxy settings differ, then there are multiple destinations that must have egress allowed. - allowedDestinations := map[string]bool{} - processedPodProxies := ProcessPodProxies(cfg.PodProxies) - for _, podProxyConfig := range processedPodProxies { - var proxyURL *url.URL - var err error - if podProxyConfig != nil && podProxyConfig.HTTPSProxy != "" { - targetURL := &url.URL{ - // The scheme should be HTTPS, as we are establishing an mTLS session with the target. - Scheme: "https", - - // We expect `target` to be of the form host:port. - Host: cfg.URL, - } - - proxyURL, err = podProxyConfig.ProxyFunc()(targetURL) - if err != nil { - return nil, err - } - } - - var tunnelDestinationHostPort string - if proxyURL != nil { - proxyHostPort, err := operatorurl.ParseHostPortFromHTTPProxyURL(proxyURL) - if err != nil { - return nil, err - } - - tunnelDestinationHostPort = proxyHostPort - } else { - // cfg.URL has host:port form - tunnelDestinationHostPort = cfg.URL - } - - // Check if we've already created an egress rule for this destination. - if allowedDestinations[tunnelDestinationHostPort] { - continue - } - - host, port, err := net.SplitHostPort(tunnelDestinationHostPort) - if err != nil { - return nil, err - } - parsedPort, err := numorstring.PortFromString(port) - if err != nil { - return nil, err - } - parsedIp := net.ParseIP(host) - if parsedIp == nil { - // Domain-based egress rules require the EgressAccessControl license feature. - if !cfg.IncludeEgressNetworkPolicy { - continue - } - // Assume host is a valid hostname. - egressRules = append(egressRules, v3.Rule{ - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: v3.EntityRule{ - Domains: []string{host}, - Ports: []numorstring.Port{parsedPort}, - }, - }) - allowedDestinations[tunnelDestinationHostPort] = true - - } else { - var netSuffix string - if parsedIp.To4() != nil { - netSuffix = "/32" - } else { - netSuffix = "/128" - } - - egressRules = append(egressRules, v3.Rule{ - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: v3.EntityRule{ - Nets: []string{parsedIp.String() + netSuffix}, - Ports: []numorstring.Port{parsedPort}, - }, - }) - allowedDestinations[tunnelDestinationHostPort] = true - } - } - - egressRules = append(egressRules, v3.Rule{Action: v3.Pass}) - - guardianIngressDestinationEntityRule := v3.EntityRule{Ports: networkpolicy.Ports(GuardianTargetPort)} - networkpolicyHelper := networkpolicy.DefaultHelper() - var ingressRules []v3.Rule - if cfg.Installation.Variant.IsEnterprise() { - ingressRules = append(ingressRules, []v3.Rule{ - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Source: FluentdSourceEntityRule, - Destination: guardianIngressDestinationEntityRule, - }, - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Source: networkpolicyHelper.ComplianceBenchmarkerSourceEntityRule(), - Destination: guardianIngressDestinationEntityRule, - }, - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Source: networkpolicyHelper.ComplianceReporterSourceEntityRule(), - Destination: guardianIngressDestinationEntityRule, - }, - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Source: networkpolicyHelper.ComplianceSnapshotterSourceEntityRule(), - Destination: guardianIngressDestinationEntityRule, - }, - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Source: networkpolicyHelper.ComplianceControllerSourceEntityRule(), - Destination: guardianIngressDestinationEntityRule, - }, - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Source: IntrusionDetectionSourceEntityRule, - Destination: guardianIngressDestinationEntityRule, - }, - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Source: IntrusionDetectionInstallerSourceEntityRule, - Destination: guardianIngressDestinationEntityRule, - }, - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: guardianIngressDestinationEntityRule, - }, - }...) - } - - policy := &v3.NetworkPolicy{ - TypeMeta: metav1.TypeMeta{Kind: "NetworkPolicy", APIVersion: "projectcalico.org/v3"}, - ObjectMeta: metav1.ObjectMeta{ - Name: GuardianPolicyName, - Namespace: GuardianNamespace, - }, - Spec: v3.NetworkPolicySpec{ - Order: &networkpolicy.HighPrecedenceOrder, - Tier: networkpolicy.CalicoTierName, - Selector: networkpolicy.KubernetesAppSelector(GuardianName), - Types: []v3.PolicyType{v3.PolicyTypeIngress, v3.PolicyTypeEgress}, - Ingress: ingressRules, - Egress: egressRules, - }, - } - - return policy, nil -} - func ProcessPodProxies(podProxies []*httpproxy.Config) []*httpproxy.Config { // If pod proxies are empty, then pod proxy resolution has not yet occurred. // Assume that a single Guardian pod is running without a proxy. diff --git a/pkg/render/guardian_test.go b/pkg/render/guardian_test.go index 7d2b454abb..244e1c6fa5 100644 --- a/pkg/render/guardian_test.go +++ b/pkg/render/guardian_test.go @@ -343,7 +343,15 @@ var _ = Describe("Rendering tests", func() { cfg.IncludeEgressNetworkPolicy = includeEgressNetworkPolicy g, err := render.GuardianPolicy(cfg) Expect(err).NotTo(HaveOccurred()) - resources, _ = g.Objects() + objs, _ := g.Objects() + // Apply the registered enterprise modifier the way the componentHandler + // does, so the enterprise policy is exercised. For the Calico variant the + // modifier is a no-op and the OSS policy is returned. + rc := extensions.RenderContext{Installation: cfg.Installation} + if p, ok := g.(render.ExtensionContextProvider); ok { + rc.Component = p.ExtensionContext() + } + resources = extensions.ApplyModifiers(render.ComponentNameGuardianPolicy, rc, objs) } Context("policy rendering based on variant and IncludeEgressNetworkPolicy", func() { From 3de36391a2024da761a21b67bb1f1d820cec5057 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Tue, 9 Jun 2026 18:50:08 -0700 Subject: [PATCH 24/38] De-variant the apiserver into an enterprise extension The apiserver was the last render component with IsEnterprise branches. Its enterprise behavior (query server container, audit logging, the tigera-* RBAC, UI settings roles, managed cluster access) moves into pkg/enterprise as the apiserver modifier, leaving render with only the shared objects. The deployment-existence check no longer keys off the variant. The controller sets RequiresQueryServer (true for Enterprise) and render branches on RequiresAggregationServer || RequiresQueryServer, so the variant decision lives in the controller rather than render. The apiserver also needs to delete enterprise objects when switching to Calico, and to drop the management-cluster RBAC when not a management cluster, so Modifier now takes and returns both the create and delete lists. The other modifiers pass the delete list through unchanged. --- .../apiserver/apiserver_controller.go | 9 +- .../apiserver/apiserver_suite_test.go | 5 + pkg/controller/utils/component.go | 2 +- pkg/controller/utils/component_test.go | 4 +- pkg/enterprise/apiserver.go | 1146 +++++++++++++++ pkg/enterprise/guardian.go | 12 +- pkg/enterprise/guardian_test.go | 10 +- pkg/enterprise/node.go | 4 +- pkg/enterprise/node_test.go | 20 +- pkg/enterprise/register.go | 1 + pkg/enterprise/typha.go | 4 +- pkg/enterprise/typha_test.go | 6 +- pkg/enterprise/windows.go | 4 +- pkg/enterprise/windows_test.go | 10 +- pkg/extensions/extension.go | 24 +- pkg/extensions/extension_test.go | 38 +- pkg/render/apiserver.go | 1269 ++--------------- pkg/render/apiserver_test.go | 117 +- pkg/render/guardian_test.go | 7 +- pkg/render/node_enterprise_test.go | 5 +- pkg/render/windows_test.go | 3 +- 21 files changed, 1413 insertions(+), 1287 deletions(-) create mode 100644 pkg/enterprise/apiserver.go diff --git a/pkg/controller/apiserver/apiserver_controller.go b/pkg/controller/apiserver/apiserver_controller.go index 668756cc10..0e762c1717 100644 --- a/pkg/controller/apiserver/apiserver_controller.go +++ b/pkg/controller/apiserver/apiserver_controller.go @@ -49,6 +49,7 @@ import ( "github.com/tigera/operator/pkg/controller/utils/imageset" "github.com/tigera/operator/pkg/ctrlruntime" "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" rcertificatemanagement "github.com/tigera/operator/pkg/render/certificatemanagement" "github.com/tigera/operator/pkg/render/common/authentication" @@ -473,8 +474,11 @@ func (r *ReconcileAPIServer) Reconcile(ctx context.Context, request reconcile.Re return reconcile.Result{}, err } - // Create a component handler to manage the rendered component. - handler := utils.NewComponentHandler(log, r.client, r.scheme, instance) + // Create a component handler to manage the rendered component. The render context + // carries the installation so the componentHandler applies the variant's API server + // modifier (query server, audit logging, Enterprise RBAC) to the rendered objects. + handler := utils.NewComponentHandler(log, r.client, r.scheme, instance, + utils.WithRenderContext(extensions.RenderContext{Installation: installationSpec})) // Render the desired objects from the CRD and create or update them. reqLogger.V(3).Info("rendering components") @@ -497,6 +501,7 @@ func (r *ReconcileAPIServer) Reconcile(ctx context.Context, request reconcile.Re KubernetesVersion: r.opts.KubernetesVersion, ClusterDomain: r.opts.ClusterDomain, RequiresAggregationServer: !r.opts.UseV3CRDs, + RequiresQueryServer: installationSpec.Variant.IsEnterprise(), QueryServerTLSKeyPairCertificateManagementOnly: queryServerTLSSecretCertificateManagementOnly, } diff --git a/pkg/controller/apiserver/apiserver_suite_test.go b/pkg/controller/apiserver/apiserver_suite_test.go index ae0a254ba0..17f3e98c8d 100644 --- a/pkg/controller/apiserver/apiserver_suite_test.go +++ b/pkg/controller/apiserver/apiserver_suite_test.go @@ -24,9 +24,14 @@ import ( logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" + + "github.com/tigera/operator/pkg/enterprise" ) func TestStatus(t *testing.T) { + // Wire the enterprise extensions so the componentHandler applies the API server + // modifier (query server, audit logging, Enterprise RBAC) during reconciliation. + enterprise.Register() logf.SetLogger(zap.New(zap.WriteTo(ginkgo.GinkgoWriter), zap.UseDevMode(true), zap.Level(uzap.NewAtomicLevelAt(uzap.DebugLevel)))) gomega.RegisterFailHandler(ginkgo.Fail) suiteConfig, reporterConfig := ginkgo.GinkgoConfiguration() diff --git a/pkg/controller/utils/component.go b/pkg/controller/utils/component.go index d2424c99c7..0eda36688b 100644 --- a/pkg/controller/utils/component.go +++ b/pkg/controller/utils/component.go @@ -473,7 +473,7 @@ func (c *componentHandler) CreateOrUpdateOrDelete(ctx context.Context, component if p, ok := component.(render.ExtensionContextProvider); ok { rc.Component = p.ExtensionContext() } - objsToCreate = extensions.ApplyModifiers(ext.ModifierKey(), rc, objsToCreate) + objsToCreate, objsToDelete = extensions.ApplyModifiers(ext.ModifierKey(), rc, objsToCreate, objsToDelete) } // Load the InstallationSpec once and reuse it for every object: createOrUpdateObject needs it diff --git a/pkg/controller/utils/component_test.go b/pkg/controller/utils/component_test.go index ecbe90afe4..fad9e6fedb 100644 --- a/pkg/controller/utils/component_test.go +++ b/pkg/controller/utils/component_test.go @@ -2494,10 +2494,10 @@ var _ = Describe("componentHandler modifier application", func() { It("applies registered modifiers to a named component before create", func() { extensions.Register(operatorv1.CalicoEnterprise, "fake", extensions.Extension{ - Modify: func(ctx extensions.RenderContext, objs []client.Object) []client.Object { + Modify: func(ctx extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { cm := objs[0].(*corev1.ConfigMap) cm.Data = map[string]string{"patched": "yes"} - return objs + return objs, del }, }) diff --git a/pkg/enterprise/apiserver.go b/pkg/enterprise/apiserver.go new file mode 100644 index 0000000000..178be099a3 --- /dev/null +++ b/pkg/enterprise/apiserver.go @@ -0,0 +1,1146 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enterprise + +import ( + "fmt" + "strings" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/render" + rcomp "github.com/tigera/operator/pkg/render/common/components" + relasticsearch "github.com/tigera/operator/pkg/render/common/elasticsearch" + rmeta "github.com/tigera/operator/pkg/render/common/meta" + "github.com/tigera/operator/pkg/render/common/securitycontext" + "github.com/tigera/operator/pkg/tls/certificatemanagement" +) + +const ( + auditLogsVolumeName = "calico-audit-logs" + auditPolicyVolumeName = "calico-audit-policy" +) + +// apiServer carries the rendered API server configuration and resolved image so the +// enterprise builders (moved verbatim from the render package) can construct the +// Enterprise-only objects and deployment additions. +type apiServer struct { + cfg *render.APIServerConfiguration + calicoImage string +} + +func registerAPIServer() { + extensions.Register(operatorv1.CalicoEnterprise, render.ComponentNameAPIServer, extensions.Extension{ + Modify: modifyAPIServer, + }) + // When running Calico, clean up any Enterprise objects left behind by a prior + // Enterprise installation. + extensions.Register(operatorv1.Calico, render.ComponentNameAPIServer, extensions.Extension{ + Modify: cleanupAPIServer, + }) +} + +// modifyAPIServer layers Calico Enterprise behavior onto the rendered API server objects: +// the query server container and its volumes, audit logging on the aggregation API server +// container, the Enterprise RBAC objects, and the query server port on the Service. +func modifyAPIServer(ctx extensions.RenderContext, create, del []client.Object) ([]client.Object, []client.Object) { + ec, _ := ctx.Component.(render.APIServerExtensionContext) + c := &apiServer{cfg: ec.Config, calicoImage: ec.CalicoImage} + + if dep, ok := extensions.FindObject[*appsv1.Deployment](create, render.APIServerName); ok { + c.layerDeployment(dep) + } + if svc, ok := extensions.FindObject[*corev1.Service](create, render.APIServerServiceName); ok { + c.addQueryServerPort(svc) + } + // Enterprise serves staged policies through the tiered-policy passthrough role. + if role, ok := extensions.FindObject[*rbacv1.ClusterRole](create, "calico-tiered-policy-passthrough"); ok { + for i := range role.Rules { + if contains(role.Rules[i].Resources, "networkpolicies") { + role.Rules[i].Resources = append(role.Rules[i].Resources, "stagednetworkpolicies", "stagedglobalnetworkpolicies") + } + } + } + + // Global Enterprise RBAC. + create = append(create, c.tigeraAPIServerClusterRole(), c.tigeraAPIServerClusterRoleBinding()) + if !c.cfg.MultiTenant { + // These resources are only installed in zero-tenant clusters. + create = append(create, c.tigeraUserClusterRole(), c.tigeraNetworkAdminClusterRole()) + } + if c.cfg.ManagementCluster != nil { + create = append(create, c.managedClusterWatchClusterRole()) + if c.cfg.MultiTenant { + create = append(create, c.multiTenantSecretsRBAC()...) + create = append(create, c.multiTenantManagedClusterAccessClusterRoles()...) + } else { + create = append(create, c.secretsRBAC()...) + } + } else { + // If we're not a management cluster, the API server doesn't need permissions to access secrets. + del = append(del, c.multiTenantSecretsRBAC()...) + del = append(del, c.secretsRBAC()...) + del = append(del, c.multiTenantManagedClusterAccessClusterRoles()...) + del = append(del, c.managedClusterWatchClusterRole()) + } + + // Namespaced Enterprise objects. + if c.cfg.TrustedBundle != nil { + create = append(create, c.cfg.TrustedBundle.ConfigMap(render.QueryserverNamespace)) + } + if c.cfg.ManagementClusterConnection != nil { + create = append(create, c.externalLinseedRoleBinding()) + } + + // Objects that only exist alongside the aggregation API server. + aggregationObjects := []client.Object{ + c.uiSettingsGroupGetterClusterRole(), + c.kubeControllerManagerUISettingsGroupGetterClusterRoleBinding(), + c.uiSettingsPassthruClusterRole(), + c.uiSettingsPassthruClusterRolebinding(), + c.auditPolicyConfigMap(), + } + if c.cfg.RequiresAggregationServer { + create = append(create, aggregationObjects...) + } else { + del = append(del, aggregationObjects...) + } + + // Clean up cluster-scoped resources that were created with the 'tigera' prefix. + del = append(del, c.deprecatedResources()...) + + // Re-apply deployment overrides so the modifier-added query server container picks up + // any per-container overrides. The override appliers use replace/merge semantics, so + // re-running over the render-applied containers is idempotent. + if dep, ok := extensions.FindObject[*appsv1.Deployment](create, render.APIServerName); ok { + if overrides := c.cfg.APIServer.APIServerDeployment; overrides != nil { + rcomp.ApplyDeploymentOverrides(dep, overrides) + } + } + + return create, del +} + +// cleanupAPIServer deletes the Enterprise API server objects when running Calico, so a +// cluster switched from Enterprise to Calico does not leave them behind. +func cleanupAPIServer(ctx extensions.RenderContext, create, del []client.Object) ([]client.Object, []client.Object) { + ec, _ := ctx.Component.(render.APIServerExtensionContext) + c := &apiServer{cfg: ec.Config} + + del = append(del, c.tigeraAPIServerClusterRole(), c.tigeraAPIServerClusterRoleBinding()) + if !c.cfg.MultiTenant { + del = append(del, c.tigeraUserClusterRole(), c.tigeraNetworkAdminClusterRole()) + } + del = append(del, c.multiTenantSecretsRBAC()...) + del = append(del, c.secretsRBAC()...) + del = append(del, c.multiTenantManagedClusterAccessClusterRoles()...) + del = append(del, c.managedClusterWatchClusterRole()) + + return create, del +} + +// layerDeployment adds the Enterprise query server container, audit logging, and the +// query server / trusted bundle volumes to the rendered API server deployment. +func (c *apiServer) layerDeployment(d *appsv1.Deployment) { + // Audit logging is performed through the aggregation API server container, which is + // only present when the aggregation API server is running. + if c.cfg.RequiresAggregationServer { + for i := range d.Spec.Template.Spec.Containers { + ctr := &d.Spec.Template.Spec.Containers[i] + if ctr.Name != string(render.APIServerContainerName) { + continue + } + ctr.VolumeMounts = append(ctr.VolumeMounts, + corev1.VolumeMount{Name: auditLogsVolumeName, MountPath: "/var/log/calico/audit"}, + corev1.VolumeMount{Name: auditPolicyVolumeName, MountPath: "/etc/tigera/audit"}, + ) + ctr.Args = append(ctr.Args, + "--audit-policy-file=/etc/tigera/audit/policy.conf", + "--audit-log-path=/var/log/calico/audit/tsee-audit.log", + ) + // In case of OpenShift, apiserver needs privileged access to write audit logs to the + // host path volume. Audit logs are owned by root on hosts so we need to be root. + ctr.SecurityContext = securitycontext.NewRootContext(c.cfg.OpenShift) + } + + d.Spec.Template.Spec.Volumes = append(d.Spec.Template.Spec.Volumes, c.auditVolumes()...) + } + + d.Spec.Template.Spec.Containers = append(d.Spec.Template.Spec.Containers, c.queryServerContainer()) + + if c.cfg.TrustedBundle != nil { + d.Spec.Template.Spec.Volumes = append(d.Spec.Template.Spec.Volumes, c.cfg.TrustedBundle.Volume()) + for k, v := range c.cfg.TrustedBundle.HashAnnotations() { + d.Spec.Template.Annotations[k] = v + } + } +} + +// auditVolumes are the host-path audit log and audit policy volumes used by the +// aggregation API server container. +func (c *apiServer) auditVolumes() []corev1.Volume { + return []corev1.Volume{ + { + Name: auditLogsVolumeName, + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/var/log/calico/audit", + Type: ptr.To(corev1.HostPathDirectoryOrCreate), + }, + }, + }, + { + Name: auditPolicyVolumeName, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: auditPolicyVolumeName}, + Items: []corev1.KeyToPath{ + { + Key: "config", + Path: "policy.conf", + }, + }, + }, + }, + }, + } +} + +func (c *apiServer) addQueryServerPort(s *corev1.Service) { + queryServerTargetPort := render.GetContainerPort(c.cfg, render.TigeraAPIServerQueryServerContainerName) + s.Spec.Ports = append(s.Spec.Ports, corev1.ServicePort{ + Name: render.QueryServerPortName, + Port: render.QueryServerPort, + Protocol: corev1.ProtocolTCP, + TargetPort: intstr.FromInt32(queryServerTargetPort.ContainerPort), + }) +} + +func contains(s []string, v string) bool { + for _, x := range s { + if x == v { + return true + } + } + return false +} + +func (c *apiServer) multiTenantSecretsRBAC() []client.Object { + return render.TunnelSecretRBAC(render.APIServerSecretsRBACName, render.APIServerServiceAccountName, c.cfg.ManagementCluster, true) +} + +func (c *apiServer) secretsRBAC() []client.Object { + return render.TunnelSecretRBAC(render.APIServerSecretsRBACName, render.APIServerServiceAccountName, c.cfg.ManagementCluster, false) +} + +func (c *apiServer) queryServerContainer() corev1.Container { + queryServerTargetPort := render.GetContainerPort(c.cfg, render.TigeraAPIServerQueryServerContainerName).ContainerPort + + var tlsSecret certificatemanagement.KeyPairInterface + if c.cfg.QueryServerTLSKeyPairCertificateManagementOnly != nil { + tlsSecret = c.cfg.QueryServerTLSKeyPairCertificateManagementOnly + } else { + tlsSecret = c.cfg.TLSKeyPair + } + env := []corev1.EnvVar{ + {Name: "DATASTORE_TYPE", Value: "kubernetes"}, + {Name: "LISTEN_ADDR", Value: fmt.Sprintf(":%d", queryServerTargetPort)}, + {Name: "TLS_CERT", Value: fmt.Sprintf("/%s/tls.crt", tlsSecret.GetName())}, + {Name: "TLS_KEY", Value: fmt.Sprintf("/%s/tls.key", tlsSecret.GetName())}, + } + if c.cfg.TrustedBundle != nil { + env = append(env, corev1.EnvVar{Name: "TRUSTED_BUNDLE_PATH", Value: c.cfg.TrustedBundle.MountPath()}) + } + + if render.HostNetwork(c.cfg) { + env = append(env, c.cfg.K8SServiceEndpoint.EnvVars()...) + } else { + env = append(env, c.cfg.K8SServiceEndpointPodNetwork.EnvVars()...) + } + + if c.cfg.Installation.CalicoNetwork != nil && c.cfg.Installation.CalicoNetwork.MultiInterfaceMode != nil { + env = append(env, corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: c.cfg.Installation.CalicoNetwork.MultiInterfaceMode.Value()}) + } + + if c.cfg.KeyValidatorConfig != nil { + env = append(env, c.cfg.KeyValidatorConfig.RequiredEnv("")...) + } + + linseedURL := relasticsearch.LinseedEndpoint(rmeta.OSTypeLinux, c.cfg.ClusterDomain, render.ElasticsearchNamespace, c.cfg.ManagementClusterConnection != nil, false) + env = append(env, + corev1.EnvVar{Name: "LINSEED_URL", Value: linseedURL}, + corev1.EnvVar{Name: "LINSEED_CLIENT_CERT", Value: fmt.Sprintf("/%s/tls.crt", tlsSecret.GetName())}, + corev1.EnvVar{Name: "LINSEED_CLIENT_KEY", Value: fmt.Sprintf("/%s/tls.key", tlsSecret.GetName())}, + ) + if c.cfg.ManagementClusterConnection != nil { + env = append(env, + corev1.EnvVar{Name: "CLUSTER_ID", Value: ""}, + corev1.EnvVar{Name: "LINSEED_TOKEN", Value: render.GetLinseedTokenPath(true)}, + ) + } + if c.cfg.TrustedBundle != nil { + env = append(env, corev1.EnvVar{Name: "LINSEED_CA", Value: c.cfg.TrustedBundle.MountPath()}) + } + + // set LogLEVEL for queryserver container + if logging := c.cfg.APIServer.Logging; logging != nil && + logging.QueryServerLogging != nil && logging.QueryServerLogging.LogSeverity != nil { + env = append(env, + corev1.EnvVar{Name: "LOGLEVEL", Value: strings.ToLower(string(*logging.QueryServerLogging.LogSeverity))}) + } else { + // set default LOGLEVEL to info when not set by the user + env = append(env, corev1.EnvVar{Name: "LOGLEVEL", Value: "info"}) + } + + volumeMounts := []corev1.VolumeMount{ + tlsSecret.VolumeMount(rmeta.OSTypeLinux), + } + if c.cfg.TrustedBundle != nil { + volumeMounts = append(volumeMounts, c.cfg.TrustedBundle.VolumeMounts(rmeta.OSTypeLinux)...) + } + if c.cfg.ManagementClusterConnection != nil { + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + Name: render.LinseedTokenVolumeName, + MountPath: render.LinseedVolumeMountPath, + }) + } + + container := corev1.Container{ + Name: string(render.TigeraAPIServerQueryServerContainerName), + Image: c.calicoImage, + Command: []string{components.CalicoBinaryPath, "component", "queryserver"}, + Env: env, + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/version", + Port: intstr.FromInt32(queryServerTargetPort), + Scheme: corev1.URISchemeHTTPS, + }, + }, + InitialDelaySeconds: 90, + }, + SecurityContext: securitycontext.NewNonRootContext(), + VolumeMounts: volumeMounts, + } + return container +} + +func (c *apiServer) externalLinseedRoleBinding() *rbacv1.RoleBinding { + return &rbacv1.RoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "tigera-linseed", + Namespace: render.APIServerNamespace, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "ClusterRole", + Name: render.TigeraLinseedSecretsClusterRole, + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: render.GuardianServiceAccountName, + Namespace: render.GuardianNamespace, + }, + }, + } +} + +func (c *apiServer) tigeraAPIServerClusterRole() *rbacv1.ClusterRole { + rules := []rbacv1.PolicyRule{ + { + // Read access to Linseed policy activity data for queryserver enrichment. + APIGroups: []string{"linseed.tigera.io"}, + Resources: []string{"policyactivity"}, + Verbs: []string{"get"}, + }, + { + // Calico Enterprise backing storage. + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{ + "alertexceptions", + "bfdconfigurations", + "deeppacketinspections", + "deeppacketinspections/status", + "egressgatewaypolicies", + "externalnetworks", + "globalalerts", + "globalalerts/status", + "globalalerttemplates", + "globalreports", + "globalreports/status", + "globalreporttypes", + "globalthreatfeeds", + "globalthreatfeeds/status", + "licensekeys", + "managedclusters", + "managedclusters/status", + "networks", + "packetcaptures", + "packetcaptures/status", + "policyrecommendationscopes", + "policyrecommendationscopes/status", + "remoteclusterconfigurations", + "securityeventwebhooks", + "securityeventwebhooks/status", + "uisettings", + "uisettingsgroups", + }, + Verbs: []string{ + "get", + "list", + "watch", + "create", + "update", + "delete", + "patch", + }, + }, + { + // The queryserver's RBAC calculator needs to list tiers, + // uisettingsgroups, and managedclusters via the aggregated + // API to evaluate user permissions for the /policies endpoint. + APIGroups: []string{"projectcalico.org"}, + Resources: []string{ + "tiers", + "uisettingsgroups", + "managedclusters", + }, + Verbs: []string{"get", "list", "watch"}, + }, + { + // Required by the AuthorizationReview calculator in queryserver to evaluate + // RBAC permissions for users. + APIGroups: []string{"rbac.authorization.k8s.io"}, + Resources: []string{ + "clusterroles", + "clusterrolebindings", + "roles", + "rolebindings", + }, + Verbs: []string{"get", "list", "watch"}, + }, + } + + return &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: render.APIServerName, + }, + Rules: rules, + } +} + +func (c *apiServer) tigeraAPIServerClusterRoleBinding() *rbacv1.ClusterRoleBinding { + return &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: render.APIServerName, + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: render.APIServerServiceAccountName, + Namespace: render.APIServerNamespace, + }, + }, + RoleRef: rbacv1.RoleRef{ + Kind: "ClusterRole", + Name: render.APIServerName, + APIGroup: "rbac.authorization.k8s.io", + }, + } +} + +func (c *apiServer) uiSettingsGroupGetterClusterRole() *rbacv1.ClusterRole { + return &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "calico-uisettingsgroup-getter", + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{ + "uisettingsgroups", + }, + Verbs: []string{"get"}, + }, + }, + } +} + +func (c *apiServer) kubeControllerManagerUISettingsGroupGetterClusterRoleBinding() *rbacv1.ClusterRoleBinding { + return &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "calico-uisettingsgroup-getter", + }, + RoleRef: rbacv1.RoleRef{ + Kind: "ClusterRole", + Name: "calico-uisettingsgroup-getter", + APIGroup: "rbac.authorization.k8s.io", + }, + Subjects: []rbacv1.Subject{ + { + Kind: "User", + Name: "system:kube-controller-manager", + APIGroup: "rbac.authorization.k8s.io", + }, + }, + } +} + +func (c *apiServer) tigeraUserClusterRole() *rbacv1.ClusterRole { + rules := []rbacv1.PolicyRule{ + // List requests that the Tigera manager needs. + { + APIGroups: []string{ + "projectcalico.org", + "networking.k8s.io", + "extensions", + "", + }, + // Use both the networkpolicies and tier.networkpolicies resource types to ensure identical behavior + // irrespective of the Calico RBAC scheme (see the ClusterRole "calico-tiered-policy-passthrough" for + // more details). Similar for all tiered policy resource types. + Resources: []string{ + "tiers", + "networkpolicies", + "tier.networkpolicies", + "globalnetworkpolicies", + "tier.globalnetworkpolicies", + "namespaces", + "globalnetworksets", + "networksets", + "managedclusters", + "stagedglobalnetworkpolicies", + "tier.stagedglobalnetworkpolicies", + "stagednetworkpolicies", + "tier.stagednetworkpolicies", + "stagedkubernetesnetworkpolicies", + "policyrecommendationscopes", + }, + Verbs: []string{"watch", "list"}, + }, + { + APIGroups: []string{"policy.networking.k8s.io"}, + Resources: []string{ + "clusternetworkpolicies", + "adminnetworkpolicies", + "baselineadminnetworkpolicies", + }, + Verbs: []string{"watch", "list"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"packetcaptures/files"}, + Verbs: []string{"get"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"packetcaptures"}, + Verbs: []string{"get", "list", "watch"}, + }, + // Additional "list" requests required to view flows. + { + APIGroups: []string{""}, + Resources: []string{"pods"}, + Verbs: []string{"list"}, + }, + // Additional "list" requests required to view serviceaccount labels. + { + APIGroups: []string{""}, + Resources: []string{"serviceaccounts"}, + Verbs: []string{"list"}, + }, + // Access for WAF API to read in coreruleset configmap + { + APIGroups: []string{""}, + Resources: []string{"configmaps"}, + ResourceNames: []string{"coreruleset-default"}, + Verbs: []string{"get"}, + }, + // Access to statistics. + { + APIGroups: []string{""}, + Resources: []string{"services/proxy"}, + ResourceNames: []string{ + "https:calico-api:8080", "calico-node-prometheus:9090", + }, + Verbs: []string{"get", "create"}, + }, + // Access to policies in all tiers + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"tiers"}, + Verbs: []string{"get"}, + }, + // List and download the reports in the Tigera Secure manager. + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"globalreports"}, + Verbs: []string{"get", "list"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"globalreporttypes"}, + Verbs: []string{"get"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"clusterinformations"}, + Verbs: []string{"get", "list"}, + }, + // Access to hostendpoints from the UI ServiceGraph. + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"hostendpoints"}, + Verbs: []string{"get", "list"}, + }, + // List and view the threat defense configuration + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{ + "alertexceptions", + "globalalerts", + "globalalerts/status", + "globalalerttemplates", + "globalthreatfeeds", + "globalthreatfeeds/status", + "securityeventwebhooks", + }, + Verbs: []string{"get", "watch", "list"}, + }, + // User can: + // - read UISettings in the cluster-settings group + // - read and write UISettings in the user-settings group + // Default settings group and settings are created in manager.go. + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"uisettingsgroups"}, + Verbs: []string{"get"}, + ResourceNames: []string{"cluster-settings", "user-settings"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"uisettingsgroups/data"}, + Verbs: []string{"get", "list", "watch"}, + ResourceNames: []string{"cluster-settings"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"uisettingsgroups/data"}, + Verbs: []string{"*"}, + ResourceNames: []string{"user-settings"}, + }, + // Allow the user to read applicationlayers to detect if WAF is enabled/disabled. + { + APIGroups: []string{"operator.tigera.io"}, + Resources: []string{"applicationlayers", "packetcaptureapis", "compliances", "intrusiondetections"}, + Verbs: []string{"get"}, + }, + { + APIGroups: []string{"apps"}, + Resources: []string{"deployments"}, + Verbs: []string{"get", "list", "watch"}, + }, + // Allow the user to read services to view WAF configuration. + { + APIGroups: []string{""}, + Resources: []string{"services"}, + Verbs: []string{"get", "list", "watch"}, + }, + // Allow the user to read felixconfigurations to detect if wireguard and/or other features are enabled. + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"felixconfigurations"}, + Verbs: []string{"get", "list"}, + }, + // Allow the user to only view securityeventwebhooks. + { + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"securityeventwebhooks"}, + Verbs: []string{"get", "list"}, + }, + } + + // Privileges for lma.tigera.io have no effect on managed clusters. + if c.cfg.ManagementClusterConnection == nil { + // Access to flow logs, audit logs, and statistics. + // Access to log into Kibana for oidc users. + rules = append(rules, rbacv1.PolicyRule{ + APIGroups: []string{"lma.tigera.io"}, + Resources: []string{"*"}, + ResourceNames: []string{ + "flows", "audit*", "l7", "events", "dns", "waf", "kibana_login", "recommendations", + }, + Verbs: []string{"get"}, + }) + } + + return &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "tigera-ui-user", + }, + Rules: rules, + } +} + +func (c *apiServer) tigeraNetworkAdminClusterRole() *rbacv1.ClusterRole { + rules := []rbacv1.PolicyRule{ + // Full access to all network policies + { + APIGroups: []string{ + "projectcalico.org", + "networking.k8s.io", + "extensions", + }, + // Use both the networkpolicies and tier.networkpolicies resource types to ensure identical behavior + // irrespective of the Calico RBAC scheme (see the ClusterRole "calico-tiered-policy-passthrough" for + // more details). Similar for all tiered policy resource types. + Resources: []string{ + "tiers", + "networkpolicies", + "tier.networkpolicies", + "globalnetworkpolicies", + "tier.globalnetworkpolicies", + "stagedglobalnetworkpolicies", + "tier.stagedglobalnetworkpolicies", + "stagednetworkpolicies", + "tier.stagednetworkpolicies", + "stagedkubernetesnetworkpolicies", + "globalnetworksets", + "networksets", + "managedclusters", + "packetcaptures", + "policyrecommendationscopes", + }, + Verbs: []string{"create", "update", "delete", "patch", "get", "watch", "list"}, + }, + { + APIGroups: []string{ + "policy.networking.k8s.io", + }, + Resources: []string{ + "clusternetworkpolicies", + "adminnetworkpolicies", + "baselineadminnetworkpolicies", + }, + Verbs: []string{"create", "update", "delete", "patch", "get", "watch", "list"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"packetcaptures/files"}, + Verbs: []string{"get", "delete"}, + }, + // Additional "list" requests that the Tigera Secure manager needs + { + APIGroups: []string{""}, + Resources: []string{"namespaces"}, + Verbs: []string{"watch", "list"}, + }, + // Additional "list" requests required to view flows. + { + APIGroups: []string{""}, + Resources: []string{"pods"}, + Verbs: []string{"list"}, + }, + // Additional "list" requests required to view serviceaccount labels. + { + APIGroups: []string{""}, + Resources: []string{"serviceaccounts"}, + Verbs: []string{"list"}, + }, + // Access for WAF API to read in coreruleset configmap + { + APIGroups: []string{""}, + Resources: []string{"configmaps"}, + ResourceNames: []string{"coreruleset-default"}, + Verbs: []string{"get"}, + }, + // Access to statistics. + { + APIGroups: []string{""}, + Resources: []string{"services/proxy"}, + ResourceNames: []string{ + "https:calico-api:8080", "calico-node-prometheus:9090", + }, + Verbs: []string{"get", "create"}, + }, + // Manage globalreport configuration, view report generation status, and list reports in the Tigera Secure manager. + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"globalreports"}, + Verbs: []string{"*"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"globalreports/status"}, + Verbs: []string{"get", "list", "watch"}, + }, + // List and download the reports in the Tigera Secure manager. + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"globalreporttypes"}, + Verbs: []string{"get"}, + }, + // Access to cluster information containing Calico and EE versions from the UI. + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"clusterinformations"}, + Verbs: []string{"get", "list"}, + }, + // Access to hostendpoints from the UI ServiceGraph. + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"hostendpoints"}, + Verbs: []string{"get", "list"}, + }, + // Manage the threat defense configuration + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{ + "alertexceptions", + "globalalerts", + "globalalerts/status", + "globalalerttemplates", + "globalthreatfeeds", + "globalthreatfeeds/status", + "securityeventwebhooks", + }, + Verbs: []string{"create", "update", "delete", "patch", "get", "watch", "list"}, + }, + // User can: + // - read and write UISettings in the cluster-settings group, and rename the group + // - read and write UISettings in the user-settings group, and rename the group + // Default settings group and settings are created in manager.go. + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"uisettingsgroups"}, + Verbs: []string{"get", "patch", "update"}, + ResourceNames: []string{"cluster-settings", "user-settings"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"uisettingsgroups/data"}, + Verbs: []string{"*"}, + ResourceNames: []string{"cluster-settings", "user-settings"}, + }, + // Allow the user to read and write applicationlayers to enable/disable WAF. + { + APIGroups: []string{"operator.tigera.io"}, + Resources: []string{"applicationlayers", "packetcaptureapis", "compliances", "intrusiondetections"}, + Verbs: []string{"get", "update", "patch", "create", "delete"}, + }, + // Allow the user to read deployments to view WAF configuration. + { + APIGroups: []string{"apps"}, + Resources: []string{"deployments"}, + Verbs: []string{"get", "list", "watch", "patch"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"services"}, + Verbs: []string{"get", "list", "watch", "patch"}, + }, + // Allow the user to read felixconfigurations to detect if wireguard and/or other features are enabled. + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"felixconfigurations"}, + Verbs: []string{"get", "list"}, + }, + // Allow the user to perform CRUD operations on securityeventwebhooks. + { + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"securityeventwebhooks"}, + Verbs: []string{"get", "list", "update", "patch", "create", "delete"}, + }, + // Allow the user to create secrets. + { + APIGroups: []string{""}, + Resources: []string{ + "secrets", + }, + Verbs: []string{"create"}, + }, + // Allow the user to patch webhooks-secret secret. + { + APIGroups: []string{""}, + Resources: []string{ + "secrets", + }, + ResourceNames: []string{ + "webhooks-secret", + }, + Verbs: []string{"patch"}, + }, + } + + // Privileges for lma.tigera.io have no effect on managed clusters. + if c.cfg.ManagementClusterConnection == nil { + // Access to flow logs, audit logs, and statistics. + // Elasticsearch superuser access once logged into Kibana. + rules = append(rules, rbacv1.PolicyRule{ + APIGroups: []string{"lma.tigera.io"}, + Resources: []string{"*"}, + ResourceNames: []string{ + "flows", "audit*", "l7", "events", "dns", "waf", "kibana_login", "elasticsearch_superuser", "recommendations", + }, + Verbs: []string{"get"}, + }) + } + + // In v3 CRD / webhooks mode there is no aggregated apiserver, and the + // calico-uisettings-passthrough ClusterRole that normally grants the broad + // uisettings permission isn't deployed. Grant write verbs here so the + // calico-webhooks UISettings handler (which narrows access via a SAR on + // uisettingsgroups/data) gets invoked instead of being short-circuited by + // kube-apiserver RBAC. + if !c.cfg.RequiresAggregationServer { + rules = append(rules, rbacv1.PolicyRule{ + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"uisettings"}, + Verbs: []string{"create", "update", "delete", "patch"}, + }) + } + + return &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "tigera-network-admin", + }, + Rules: rules, + } +} + +func (c *apiServer) uiSettingsPassthruClusterRole() *rbacv1.ClusterRole { + return &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "calico-uisettings-passthrough", + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"uisettings"}, + Verbs: []string{"*"}, + }, + }, + } +} + +func (c *apiServer) uiSettingsPassthruClusterRolebinding() *rbacv1.ClusterRoleBinding { + return &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "calico-uisettings-passthrough", + }, + Subjects: []rbacv1.Subject{ + { + Kind: "Group", + Name: "system:authenticated", + APIGroup: "rbac.authorization.k8s.io", + }, + }, + RoleRef: rbacv1.RoleRef{ + Kind: "ClusterRole", + Name: "calico-uisettings-passthrough", + APIGroup: "rbac.authorization.k8s.io", + }, + } +} + +func (c *apiServer) auditPolicyConfigMap() *corev1.ConfigMap { + const defaultAuditPolicy = `apiVersion: audit.k8s.io/v1 +kind: Policy +rules: +- level: RequestResponse + omitStages: + - RequestReceived + verbs: + - create + - patch + - update + - delete + resources: + - group: projectcalico.org + resources: + - globalnetworkpolicies + - networkpolicies + - stagedglobalnetworkpolicies + - stagednetworkpolicies + - stagedkubernetesnetworkpolicies + - globalnetworksets + - networksets + - tiers + - hostendpoints` + + return &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + // This object is for Enterprise only, so pass it explicitly. + Namespace: render.APIServerNamespace, + Name: auditPolicyVolumeName, + }, + Data: map[string]string{ + "config": defaultAuditPolicy, + }, + } +} + +func (c *apiServer) multiTenantManagedClusterAccessClusterRoles() []client.Object { + var objects []client.Object + objects = append(objects, &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: render.MultiTenantManagedClustersAccessClusterRoleName}, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"managedclusters"}, + Verbs: []string{ + // The Authentication Proxy in Voltron checks if Enterprise Components (using impersonation headers for + // the service in the canonical namespace) can get a managed clusters before sending the request down the tunnel. + // This ClusterRole will be assigned to each component using a RoleBinding in the canonical or tenant namespace. + "get", + }, + }, + }, + }) + + return objects +} + +func (c *apiServer) managedClusterWatchClusterRole() client.Object { + return &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: render.ManagedClustersWatchClusterRoleName}, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"managedclusters"}, + Verbs: []string{ + "get", "list", "watch", + }, + }, + }, + } +} + +func (c *apiServer) deprecatedResources() []client.Object { + return []client.Object{ + &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-extension-apiserver-secrets-access"}, + }, + &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-extension-apiserver-secrets-access"}, + }, + + // delegateAuthClusterRoleBinding + &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-apiserver-delegate-auth"}, + }, + + // authClusterRole + &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-extension-apiserver-auth-access"}, + }, + + // authClusterRoleBinding + &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-extension-apiserver-auth-access"}, + }, + // authReaderRoleBinding - need clean up in diff namespace kube-system + &rbacv1.RoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "tigera-auth-reader", + Namespace: "kube-system", + }, + }, + // webhookReaderClusterRole + &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-webhook-reader"}, + }, + + // webhookReaderClusterRoleBinding + &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-apiserver-webhook-reader"}, + }, + + // calico-apiserver CR and CRB + &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-apiserver"}, + }, + &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-apiserver"}, + }, + + &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-uisettingsgroup-getter"}, + }, + &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-uisettingsgroup-getter"}, + }, + + &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-tiered-policy-passthrough"}, + }, + &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-tiered-policy-passthrough"}, + }, + + &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-uisettings-passthrough"}, + }, + &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-uisettings-passthrough"}, + }, + + // Clean up legacy secrets in the tigera-operator namespace + &corev1.Secret{ + TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-api-cert", Namespace: common.OperatorNamespace()}, + }, + } +} diff --git a/pkg/enterprise/guardian.go b/pkg/enterprise/guardian.go index c85bad2680..1d16e13b19 100644 --- a/pkg/enterprise/guardian.go +++ b/pkg/enterprise/guardian.go @@ -51,21 +51,21 @@ func registerGuardian() { // enterprise management-cluster policy. Building the enterprise egress rules can // fail (proxy URL parsing); on failure we drop the policy entirely, matching the // core behavior of omitting it rather than installing a partial policy. -func modifyGuardianPolicy(ctx extensions.RenderContext, objs []client.Object) []client.Object { +func modifyGuardianPolicy(ctx extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { gpc, _ := ctx.Component.(render.GuardianPolicyExtensionContext) policy, ok := extensions.FindObject[*v3.NetworkPolicy](objs, render.GuardianPolicyName) if !ok { - return objs + return objs, del } spec, err := enterpriseGuardianPolicySpec(gpc) if err != nil { logrus.WithError(err).Error("Failed to build guardian network policy, policy will be omitted") - return removeObject(objs, policy) + return removeObject(objs, policy), del } policy.Spec = spec - return objs + return objs, del } func removeObject(objs []client.Object, drop client.Object) []client.Object { @@ -211,7 +211,7 @@ func enterpriseGuardianPolicySpec(gpc render.GuardianPolicyExtensionContext) (v3 // objects: the secrets Role/RoleBinding and default UI settings, the // elasticsearch/kibana service ports, the management-cluster-request cluster // role rules (which replace the OSS rules), and the CA bundle env vars. -func modifyGuardian(ctx extensions.RenderContext, objs []client.Object) []client.Object { +func modifyGuardian(ctx extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { gc, _ := ctx.Component.(render.GuardianExtensionContext) if role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, render.GuardianClusterRoleName); ok { @@ -234,7 +234,7 @@ func modifyGuardian(ctx extensions.RenderContext, objs []client.Object) []client render.ManagerUserSpecificSettingsGroup(), render.ManagerClusterWideTigeraLayer(), render.ManagerClusterWideDefaultView(), - ) + ), del } // guardianEnterpriseRules are the cluster role rules guardian needs in Calico diff --git a/pkg/enterprise/guardian_test.go b/pkg/enterprise/guardian_test.go index b68ad61b04..9b43423199 100644 --- a/pkg/enterprise/guardian_test.go +++ b/pkg/enterprise/guardian_test.go @@ -60,7 +60,7 @@ var _ = Describe("guardian enterprise modifier", func() { } It("appends the secrets RBAC and UI settings", func() { - out := extensions.ApplyModifiers(render.GuardianName, ctxWith(render.GuardianExtensionContext{}), newObjs()) + out, _ := extensions.ApplyModifiers(render.GuardianName, ctxWith(render.GuardianExtensionContext{}), newObjs(), nil) _, ok := extensions.FindObject[*rbacv1.Role](out, render.GuardianSecretsRole) Expect(ok).To(BeTrue()) _, ok = extensions.FindObject[*rbacv1.RoleBinding](out, render.GuardianSecretsRoleBindingName) @@ -70,7 +70,7 @@ var _ = Describe("guardian enterprise modifier", func() { }) It("adds the elasticsearch and kibana service ports", func() { - out := extensions.ApplyModifiers(render.GuardianName, ctxWith(render.GuardianExtensionContext{}), newObjs()) + out, _ := extensions.ApplyModifiers(render.GuardianName, ctxWith(render.GuardianExtensionContext{}), newObjs(), nil) svc, _ := extensions.FindObject[*corev1.Service](out, render.GuardianServiceName) names := []string{} for _, p := range svc.Spec.Ports { @@ -83,7 +83,7 @@ var _ = Describe("guardian enterprise modifier", func() { gc := render.GuardianExtensionContext{ Impersonation: &operatorv1.Impersonation{Users: []string{"foo"}, Groups: []string{"bar"}}, } - out := extensions.ApplyModifiers(render.GuardianName, ctxWith(gc), newObjs()) + out, _ := extensions.ApplyModifiers(render.GuardianName, ctxWith(gc), newObjs(), nil) role, _ := extensions.FindObject[*rbacv1.ClusterRole](out, render.GuardianClusterRoleName) // The single OSS placeholder rule is gone, replaced by the enterprise set. @@ -94,14 +94,14 @@ var _ = Describe("guardian enterprise modifier", func() { It("adds the CA bundle env to the guardian container", func() { gc := render.GuardianExtensionContext{TrustedBundleMountPath: "/ca/bundle"} - out := extensions.ApplyModifiers(render.GuardianName, ctxWith(gc), newObjs()) + out, _ := extensions.ApplyModifiers(render.GuardianName, ctxWith(gc), newObjs(), nil) dep, _ := extensions.FindObject[*appsv1.Deployment](out, render.GuardianDeploymentName) Expect(dep.Spec.Template.Spec.Containers[0].Env).To(ContainElement(corev1.EnvVar{Name: "GUARDIAN_PROMETHEUS_CA_BUNDLE_PATH", Value: "/ca/bundle"})) }) It("does nothing for the Calico variant", func() { ctx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}} - out := extensions.ApplyModifiers(render.GuardianName, ctx, newObjs()) + out, _ := extensions.ApplyModifiers(render.GuardianName, ctx, newObjs(), nil) Expect(out).To(HaveLen(len(newObjs()))) role, _ := extensions.FindObject[*rbacv1.ClusterRole](out, render.GuardianClusterRoleName) Expect(role.Rules).To(Equal([]rbacv1.PolicyRule{{Verbs: []string{"get"}}})) diff --git a/pkg/enterprise/node.go b/pkg/enterprise/node.go index 8e1de4832b..d1f2023563 100644 --- a/pkg/enterprise/node.go +++ b/pkg/enterprise/node.go @@ -59,7 +59,7 @@ func registerNode() { // objects: the extra RBAC rules, the node-metrics Service, and the Enterprise // daemonset configuration (flow/DNS log env, prometheus reporter, BGP metrics // readiness check, multi-interface mode, and the calico log volume). -func modifyNode(ctx extensions.RenderContext, objs []client.Object) []client.Object { +func modifyNode(ctx extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { if role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, render.CalicoNodeObjectName); ok { role.Rules = append(role.Rules, nodeEnterpriseRules()...) } @@ -77,7 +77,7 @@ func modifyNode(ctx extensions.RenderContext, objs []client.Object) []client.Obj modifyNodeDaemonSet(ctx, ds) } - return append(objs, nodeMetricsService(ctx)) + return append(objs, nodeMetricsService(ctx)), del } // nodeEnterpriseRules are the additional cluster role rules calico/node needs in diff --git a/pkg/enterprise/node_test.go b/pkg/enterprise/node_test.go index 2ec7e53b38..22a5408754 100644 --- a/pkg/enterprise/node_test.go +++ b/pkg/enterprise/node_test.go @@ -86,7 +86,7 @@ var _ = Describe("node enterprise modifier", func() { } It("adds the enterprise cluster role rules", func() { - out := extensions.ApplyModifiers(render.ComponentNameNode, entCtx(), newObjs()) + out, _ := extensions.ApplyModifiers(render.ComponentNameNode, entCtx(), newObjs(), nil) nodeRole, ok := extensions.FindObject[*rbacv1.ClusterRole](out, render.CalicoNodeObjectName) Expect(ok).To(BeTrue()) @@ -98,7 +98,7 @@ var _ = Describe("node enterprise modifier", func() { }) It("adds the enterprise felix env to the node container", func() { - out := extensions.ApplyModifiers(render.ComponentNameNode, entCtx(), newObjs()) + out, _ := extensions.ApplyModifiers(render.ComponentNameNode, entCtx(), newObjs(), nil) ds, _ := extensions.FindObject[*appsv1.DaemonSet](out, common.NodeDaemonSetName) c := nodeContainer(ds) @@ -115,13 +115,13 @@ var _ = Describe("node enterprise modifier", func() { ctx := entCtx() ctx.FelixConfiguration = &v3.FelixConfiguration{Spec: v3.FelixConfigurationSpec{PrometheusReporterPort: &reporter}} - out := extensions.ApplyModifiers(render.ComponentNameNode, ctx, newObjs()) + out, _ := extensions.ApplyModifiers(render.ComponentNameNode, ctx, newObjs(), nil) ds, _ := extensions.FindObject[*appsv1.DaemonSet](out, common.NodeDaemonSetName) Expect(nodeContainer(ds).Env).To(ContainElement(corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERPORT", Value: "7081"})) }) It("appends the BGP metrics readiness check when the bird check is present", func() { - out := extensions.ApplyModifiers(render.ComponentNameNode, entCtx(), newObjs()) + out, _ := extensions.ApplyModifiers(render.ComponentNameNode, entCtx(), newObjs(), nil) ds, _ := extensions.FindObject[*appsv1.DaemonSet](out, common.NodeDaemonSetName) Expect(nodeContainer(ds).ReadinessProbe.Exec.Command).To(ContainElement("--bgp-metrics-ready")) }) @@ -131,7 +131,7 @@ var _ = Describe("node enterprise modifier", func() { ds := objs[2].(*appsv1.DaemonSet) ds.Spec.Template.Spec.Containers[0].ReadinessProbe.Exec.Command = []string{"/bin/calico-node", "--felix-ready"} - out := extensions.ApplyModifiers(render.ComponentNameNode, entCtx(), objs) + out, _ := extensions.ApplyModifiers(render.ComponentNameNode, entCtx(), objs, nil) got, _ := extensions.FindObject[*appsv1.DaemonSet](out, common.NodeDaemonSetName) Expect(nodeContainer(got).ReadinessProbe.Exec.Command).NotTo(ContainElement("--bgp-metrics-ready")) }) @@ -141,7 +141,7 @@ var _ = Describe("node enterprise modifier", func() { ctx := entCtx() ctx.Installation.CalicoNetwork = &operatorv1.CalicoNetworkSpec{MultiInterfaceMode: &mode} - out := extensions.ApplyModifiers(render.ComponentNameNode, ctx, newObjs()) + out, _ := extensions.ApplyModifiers(render.ComponentNameNode, ctx, newObjs(), nil) ds, _ := extensions.FindObject[*appsv1.DaemonSet](out, common.NodeDaemonSetName) want := corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: mode.Value()} @@ -150,7 +150,7 @@ var _ = Describe("node enterprise modifier", func() { }) It("appends the node metrics service", func() { - out := extensions.ApplyModifiers(render.ComponentNameNode, entCtx(), newObjs()) + out, _ := extensions.ApplyModifiers(render.ComponentNameNode, entCtx(), newObjs(), nil) svc, ok := extensions.FindObject[*corev1.Service](out, render.CalicoNodeMetricsService) Expect(ok).To(BeTrue()) Expect(svc.Spec.Ports).To(HaveLen(2)) @@ -169,7 +169,7 @@ var _ = Describe("node enterprise modifier", func() { PrometheusMetricsEnabled: &enabled, }} - out := extensions.ApplyModifiers(render.ComponentNameNode, ctx, newObjs()) + out, _ := extensions.ApplyModifiers(render.ComponentNameNode, ctx, newObjs(), nil) svc, _ := extensions.FindObject[*corev1.Service](out, render.CalicoNodeMetricsService) Expect(svc.Spec.Ports).To(HaveLen(3)) Expect(svc.Spec.Ports[0].Port).To(Equal(int32(7081))) @@ -179,7 +179,7 @@ var _ = Describe("node enterprise modifier", func() { It("is a no-op for the Calico variant", func() { ctx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}} - out := extensions.ApplyModifiers(render.ComponentNameNode, ctx, newObjs()) + out, _ := extensions.ApplyModifiers(render.ComponentNameNode, ctx, newObjs(), nil) _, ok := extensions.FindObject[*corev1.Service](out, render.CalicoNodeMetricsService) Expect(ok).To(BeFalse()) @@ -188,7 +188,7 @@ var _ = Describe("node enterprise modifier", func() { }) It("does not panic on a zero RenderContext", func() { - out := extensions.ApplyModifiers(render.ComponentNameNode, extensions.RenderContext{}, newObjs()) + out, _ := extensions.ApplyModifiers(render.ComponentNameNode, extensions.RenderContext{}, newObjs(), nil) _, ok := extensions.FindObject[*corev1.Service](out, render.CalicoNodeMetricsService) Expect(ok).To(BeFalse()) }) diff --git a/pkg/enterprise/register.go b/pkg/enterprise/register.go index 20589ad0fa..8d6bdc0cd8 100644 --- a/pkg/enterprise/register.go +++ b/pkg/enterprise/register.go @@ -23,4 +23,5 @@ func Register() { registerWindows() registerGuardian() registerInstallation() + registerAPIServer() } diff --git a/pkg/enterprise/typha.go b/pkg/enterprise/typha.go index 457243842a..38733c871c 100644 --- a/pkg/enterprise/typha.go +++ b/pkg/enterprise/typha.go @@ -31,7 +31,7 @@ func registerTypha() { }) } -func modifyTypha(ctx extensions.RenderContext, objs []client.Object) []client.Object { +func modifyTypha(ctx extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { if role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, "calico-typha"); ok { role.Rules = append(role.Rules, rbacv1.PolicyRule{ APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, @@ -61,5 +61,5 @@ func modifyTypha(ctx extensions.RenderContext, objs []client.Object) []client.Ob } } - return objs + return objs, del } diff --git a/pkg/enterprise/typha_test.go b/pkg/enterprise/typha_test.go index 13fcccc2f9..2ebb853f8c 100644 --- a/pkg/enterprise/typha_test.go +++ b/pkg/enterprise/typha_test.go @@ -54,7 +54,7 @@ var _ = Describe("typha enterprise modifier", func() { Variant: operatorv1.CalicoEnterprise, CalicoNetwork: &operatorv1.CalicoNetworkSpec{MultiInterfaceMode: &multiMode}, }} - out := extensions.ApplyModifiers(render.ComponentNameTypha, ctx, newObjs()) + out, _ := extensions.ApplyModifiers(render.ComponentNameTypha, ctx, newObjs(), nil) role := out[0].(*rbacv1.ClusterRole) Expect(role.Rules).To(ContainElement(HaveField("Resources", ContainElement("licensekeys")))) @@ -74,14 +74,14 @@ var _ = Describe("typha enterprise modifier", func() { Variant: operatorv1.Calico, CalicoNetwork: &operatorv1.CalicoNetworkSpec{MultiInterfaceMode: &multiMode}, }} - out := extensions.ApplyModifiers(render.ComponentNameTypha, ctx, newObjs()) + out, _ := extensions.ApplyModifiers(render.ComponentNameTypha, ctx, newObjs(), nil) Expect(out[0].(*rbacv1.ClusterRole).Rules).To(BeEmpty()) dep := out[1].(*appsv1.Deployment) Expect(dep.Spec.Template.Spec.Containers[0].Env).To(BeEmpty()) }) It("does not panic on a zero Context (nil Installation)", func() { - out := extensions.ApplyModifiers(render.ComponentNameTypha, extensions.RenderContext{}, newObjs()) + out, _ := extensions.ApplyModifiers(render.ComponentNameTypha, extensions.RenderContext{}, newObjs(), nil) Expect(out[0].(*rbacv1.ClusterRole).Rules).To(BeEmpty()) }) }) diff --git a/pkg/enterprise/windows.go b/pkg/enterprise/windows.go index 12f62cd6c6..720ea80c20 100644 --- a/pkg/enterprise/windows.go +++ b/pkg/enterprise/windows.go @@ -51,14 +51,14 @@ func registerWindows() { // calico-node-windows objects: the node-metrics Service and the Enterprise // daemonset configuration (flow/DNS log env, prometheus reporter, trusted DNS // servers, the calico log volume, and the prometheus reporter keypair mount). -func modifyWindows(ctx extensions.RenderContext, objs []client.Object) []client.Object { +func modifyWindows(ctx extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { wc, _ := ctx.Component.(render.WindowsExtensionContext) if ds, ok := extensions.FindObject[*appsv1.DaemonSet](objs, common.WindowsDaemonSetName); ok { modifyWindowsDaemonSet(ctx, wc, ds) } - return append(objs, windowsNodeMetricsService(wc)) + return append(objs, windowsNodeMetricsService(wc)), del } func modifyWindowsDaemonSet(ctx extensions.RenderContext, wc render.WindowsExtensionContext, ds *appsv1.DaemonSet) { diff --git a/pkg/enterprise/windows_test.go b/pkg/enterprise/windows_test.go index b289941bdb..18668c9079 100644 --- a/pkg/enterprise/windows_test.go +++ b/pkg/enterprise/windows_test.go @@ -102,7 +102,7 @@ var _ = Describe("windows enterprise modifier", func() { } It("appends the node-metrics service", func() { - out := extensions.ApplyModifiers(render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone, nil, nil), newObjs()) + out, _ := extensions.ApplyModifiers(render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone, nil, nil), newObjs(), nil) svc, ok := extensions.FindObject[*corev1.Service](out, render.WindowsNodeMetricsService) Expect(ok).To(BeTrue()) Expect(svc.Namespace).To(Equal(common.CalicoNamespace)) @@ -110,7 +110,7 @@ var _ = Describe("windows enterprise modifier", func() { }) It("swaps the cni log mount for the calico log volume and adds enterprise env", func() { - out := extensions.ApplyModifiers(render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone, nil, nil), newObjs()) + out, _ := extensions.ApplyModifiers(render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone, nil, nil), newObjs(), nil) d := ds(out) Expect(d.Spec.Template.Spec.Volumes).To(ContainElement(HaveField("Name", "var-log-calico"))) @@ -127,7 +127,7 @@ var _ = Describe("windows enterprise modifier", func() { }) It("sets the trusted DNS server on openshift", func() { - out := extensions.ApplyModifiers(render.ComponentNameWindows, ctxFor(operatorv1.ProviderOpenShift, nil, nil), newObjs()) + out, _ := extensions.ApplyModifiers(render.ComponentNameWindows, ctxFor(operatorv1.ProviderOpenShift, nil, nil), newObjs(), nil) Expect(container(ds(out), "node").Env).To(ContainElement(corev1.EnvVar{Name: "FELIX_DNSTRUSTEDSERVERS", Value: "k8s-service:openshift-dns/dns-default"})) }) @@ -141,7 +141,7 @@ var _ = Describe("windows enterprise modifier", func() { Expect(err).NotTo(HaveOccurred()) bundle := cm.CreateTrustedBundle() - out := extensions.ApplyModifiers(render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone, tls, bundle), newObjs()) + out, _ := extensions.ApplyModifiers(render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone, tls, bundle), newObjs(), nil) d := ds(out) Expect(d.Spec.Template.Spec.Volumes).To(ContainElement(tls.Volume())) @@ -152,7 +152,7 @@ var _ = Describe("windows enterprise modifier", func() { It("does nothing for the Calico variant", func() { ctx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}} - out := extensions.ApplyModifiers(render.ComponentNameWindows, ctx, newObjs()) + out, _ := extensions.ApplyModifiers(render.ComponentNameWindows, ctx, newObjs(), nil) _, ok := extensions.FindObject[*corev1.Service](out, render.WindowsNodeMetricsService) Expect(ok).To(BeFalse()) Expect(ds(out).Spec.Template.Spec.Volumes).To(BeEmpty()) diff --git a/pkg/extensions/extension.go b/pkg/extensions/extension.go index 4f57d64dd2..25a43feb13 100644 --- a/pkg/extensions/extension.go +++ b/pkg/extensions/extension.go @@ -35,11 +35,13 @@ type Extension struct { Modify Modifier } -// Modifier post-processes the objects a render component produced. It may mutate -// matched objects and/or append additional objects, and must return the -// (possibly extended) slice. A modifier runs only for the variant it was -// registered under, so it need not re-check the variant. -type Modifier func(ctx RenderContext, objs []client.Object) []client.Object +// Modifier post-processes the objects a render component produced. It receives +// the component's create and delete lists and returns the (possibly extended) +// lists. A modifier may mutate matched objects, append objects to create, and +// append objects to delete (e.g. to clean up resources another variant left +// behind). A modifier runs only for the variant it was registered under, so it +// need not re-check the variant. +type Modifier func(ctx RenderContext, create, delete []client.Object) (newCreate, newDelete []client.Object) type modifierKey struct { variant operatorv1.ProductVariant @@ -63,16 +65,16 @@ func Register(variant operatorv1.ProductVariant, component string, e Extension) } // ApplyModifiers runs the modifier registered for the named component and the -// installation's variant over objs, returning objs unchanged when none is -// registered (or when no installation is set). -func ApplyModifiers(component string, ctx RenderContext, objs []client.Object) []client.Object { +// installation's variant over the create and delete lists, returning them +// unchanged when none is registered (or when no installation is set). +func ApplyModifiers(component string, ctx RenderContext, create, delete []client.Object) ([]client.Object, []client.Object) { if ctx.Installation == nil { - return objs + return create, delete } if fn, ok := modifiers[modifierKey{ctx.Installation.Variant, component}]; ok { - objs = fn(ctx, objs) + create, delete = fn(ctx, create, delete) } - return objs + return create, delete } // FindObject returns the first object of type T with the given name. diff --git a/pkg/extensions/extension_test.go b/pkg/extensions/extension_test.go index c935dac9f0..2c08569355 100644 --- a/pkg/extensions/extension_test.go +++ b/pkg/extensions/extension_test.go @@ -34,16 +34,16 @@ var _ = Describe("extension registry", func() { It("applies a registered modifier to the matching component and variant", func() { extensions.Register(operatorv1.CalicoEnterprise, "test", extensions.Extension{ - Modify: func(ctx extensions.RenderContext, objs []client.Object) []client.Object { + Modify: func(ctx extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { cm, ok := extensions.FindObject[*corev1.ConfigMap](objs, "cm") Expect(ok).To(BeTrue()) cm.Data = map[string]string{"k": "v"} - return append(objs, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "extra"}}) + return append(objs, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "extra"}}), del }, }) in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} - out := extensions.ApplyModifiers("test", entCtx, in) + out, _ := extensions.ApplyModifiers("test", entCtx, in, nil) Expect(out).To(HaveLen(2)) cm := out[0].(*corev1.ConfigMap) @@ -51,41 +51,57 @@ var _ = Describe("extension registry", func() { Expect(out[1].GetName()).To(Equal("extra")) }) + It("lets a modifier append to the delete list", func() { + extensions.Register(operatorv1.CalicoEnterprise, "test", extensions.Extension{ + Modify: func(_ extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { + return objs, append(del, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "stale"}}) + }, + }) + + in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} + out, del := extensions.ApplyModifiers("test", entCtx, in, nil) + Expect(out).To(Equal(in)) + Expect(del).To(HaveLen(1)) + Expect(del[0].GetName()).To(Equal("stale")) + }) + It("returns objects unchanged when no modifier is registered", func() { in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} - out := extensions.ApplyModifiers("unregistered", entCtx, in) + out, _ := extensions.ApplyModifiers("unregistered", entCtx, in, nil) Expect(out).To(Equal(in)) }) It("does not apply a modifier registered for a different variant", func() { extensions.Register(operatorv1.CalicoEnterprise, "test", extensions.Extension{ - Modify: func(_ extensions.RenderContext, objs []client.Object) []client.Object { - return append(objs, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "extra"}}) + Modify: func(_ extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { + return append(objs, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "extra"}}), del }, }) calicoCtx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}} in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} - Expect(extensions.ApplyModifiers("test", calicoCtx, in)).To(Equal(in)) + out, _ := extensions.ApplyModifiers("test", calicoCtx, in, nil) + Expect(out).To(Equal(in)) }) It("returns objects unchanged when no installation is set", func() { in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} - Expect(extensions.ApplyModifiers("test", extensions.RenderContext{}, in)).To(Equal(in)) + out, _ := extensions.ApplyModifiers("test", extensions.RenderContext{}, in, nil) + Expect(out).To(Equal(in)) }) It("replaces rather than stacks when a (variant, component) is registered twice", func() { add := func(name string) extensions.Extension { return extensions.Extension{ - Modify: func(_ extensions.RenderContext, objs []client.Object) []client.Object { - return append(objs, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: name}}) + Modify: func(_ extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { + return append(objs, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: name}}), del }, } } extensions.Register(operatorv1.CalicoEnterprise, "test", add("first")) extensions.Register(operatorv1.CalicoEnterprise, "test", add("second")) - out := extensions.ApplyModifiers("test", entCtx, nil) + out, _ := extensions.ApplyModifiers("test", entCtx, nil, nil) Expect(out).To(HaveLen(1)) Expect(out[0].GetName()).To(Equal("second")) }) diff --git a/pkg/render/apiserver.go b/pkg/render/apiserver.go index aed69c6c5f..9702226fd3 100644 --- a/pkg/render/apiserver.go +++ b/pkg/render/apiserver.go @@ -40,7 +40,6 @@ import ( "github.com/tigera/operator/pkg/controller/k8sapi" "github.com/tigera/operator/pkg/render/common/authentication" rcomp "github.com/tigera/operator/pkg/render/common/components" - relasticsearch "github.com/tigera/operator/pkg/render/common/elasticsearch" rmeta "github.com/tigera/operator/pkg/render/common/meta" "github.com/tigera/operator/pkg/render/common/networkpolicy" "github.com/tigera/operator/pkg/render/common/podaffinity" @@ -57,8 +56,9 @@ const ( APIServerPortName = "apiserver" APIServerPolicyName = networkpolicy.CalicoComponentPolicyPrefix + "apiserver-access" - auditLogsVolumeName = "calico-audit-logs" - auditPolicyVolumeName = "calico-audit-policy" + // ComponentNameAPIServer is the extension key under which a variant registers + // its API server modifier and image override. + ComponentNameAPIServer = "apiserver" ) const ( @@ -147,6 +147,12 @@ type APIServerConfiguration struct { // as part of this component. RequiresAggregationServer bool + // Whether or not the API server deployment must run a query server alongside the API + // server. The deployment (and its supporting objects) are rendered when either an + // aggregation API server or a query server is required. The query server itself, and + // the rest of its supporting configuration, is layered on by the variant's modifier. + RequiresQueryServer bool + // When certificate management is enabled, we need a separate init container to create a cert, running // with the same permissions as query server. QueryServerTLSKeyPairCertificateManagementOnly certificatemanagement.KeyPairInterface @@ -159,6 +165,25 @@ type apiServerComponent struct { dikastesImage string } +// APIServerExtensionContext carries the API server's render configuration and resolved +// image to a variant modifier. The modifier uses these to build variant-specific objects +// and to layer additional containers, volumes, and configuration onto the rendered +// deployment. +type APIServerExtensionContext struct { + Config *APIServerConfiguration + CalicoImage string +} + +// ModifierKey implements render.Extensible: the API server's variant-specific objects are +// applied by the modifier registered under this key. +func (c *apiServerComponent) ModifierKey() string { return ComponentNameAPIServer } + +// ExtensionContext implements render.ExtensionContextProvider, handing the modifier the +// config and resolved image it needs. +func (c *apiServerComponent) ExtensionContext() any { + return APIServerExtensionContext{Config: c.cfg, CalicoImage: c.calicoImage} +} + func (c *apiServerComponent) ResolveImages(is *operatorv1.ImageSet) error { reg := c.cfg.Installation.Registry path := c.cfg.Installation.ImagePath @@ -166,15 +191,14 @@ func (c *apiServerComponent) ResolveImages(is *operatorv1.ImageSet) error { var err error errMsgs := []string{} - enterprise := c.cfg.Installation.Variant.IsEnterprise() - if enterprise || c.cfg.RequiresAggregationServer { + if c.cfg.RequiresAggregationServer || c.cfg.RequiresQueryServer { c.calicoImage, err = components.GetReference(components.CombinedCalicoImage(c.cfg.Installation), reg, path, prefix, is) if err != nil { errMsgs = append(errMsgs, err.Error()) } } - if enterprise && c.cfg.IsSidecarInjectionEnabled() { + if c.cfg.IsSidecarInjectionEnabled() { c.l7AdmissionControllerEnvoyImage, err = components.GetReference(components.ComponentEnvoyProxy, reg, path, prefix, is) if err != nil { errMsgs = append(errMsgs, err.Error()) @@ -196,8 +220,8 @@ func (c *apiServerComponent) SupportedOSType() rmeta.OSType { } func (c *apiServerComponent) Objects() ([]client.Object, []client.Object) { - // Start with all of the cluster-scoped resources that are used for both Calico and Calico Enterprise. - // When switching between Calico / Enterprise, these objects are simply updated in-place. + // Cluster-scoped resources used by the API server, independent of variant. Any + // variant-specific objects are layered on by the variant's modifier. globalObjects := []client.Object{ c.calicoCustomResourcesClusterRole(), c.calicoCustomResourcesClusterRoleBinding(), @@ -209,9 +233,6 @@ func (c *apiServerComponent) Objects() ([]client.Object, []client.Object) { } objsToDelete := []client.Object{} - - // Namespaced objects common to both Calico and Calico Enterprise. - // These objects will be updated when switching between the variants. namespacedObjects := []client.Object{} // Add in image pull secrets. @@ -219,8 +240,8 @@ func (c *apiServerComponent) Objects() ([]client.Object, []client.Object) { namespacedObjects = append(namespacedObjects, secret.ToRuntimeObjects(secrets...)...) // The deployment and its supporting objects are needed when running the aggregation API server - // or when running Enterprise (which always needs the queryserver). - if c.cfg.RequiresAggregationServer || c.cfg.Installation.Variant.IsEnterprise() { + // or when a query server runs alongside it (the query server is added by a variant modifier). + if c.cfg.RequiresAggregationServer || c.cfg.RequiresQueryServer { namespacedObjects = append(namespacedObjects, c.apiServerServiceAccount(), c.apiServerDeployment(), @@ -246,16 +267,6 @@ func (c *apiServerComponent) Objects() ([]client.Object, []client.Object) { c.authReaderRoleBinding(), } - if c.cfg.Installation.Variant.IsEnterprise() { - aggregationAPIServerObjects = append(aggregationAPIServerObjects, - c.uiSettingsGroupGetterClusterRole(), - c.kubeControllerManagerUISettingsGroupGetterClusterRoleBinding(), - c.uiSettingsPassthruClusterRole(), - c.uiSettingsPassthruClusterRolebinding(), - c.auditPolicyConfigMap(), - ) - } - // Add in certificates for API server TLS. if !c.cfg.TLSKeyPair.UseCertificateManagement() { aggregationAPIServerObjects = append(aggregationAPIServerObjects, c.apiServiceRegistration(c.cfg.TLSKeyPair.GetCertificatePEM())) @@ -263,73 +274,12 @@ func (c *apiServerComponent) Objects() ([]client.Object, []client.Object) { aggregationAPIServerObjects = append(aggregationAPIServerObjects, c.apiServiceRegistration(c.cfg.Installation.CertificateManagement.CACert)) } - // Global enterprise-only objects. - globalEnterpriseObjects := []client.Object{ - c.tigeraAPIServerClusterRole(), - c.tigeraAPIServerClusterRoleBinding(), - } - - if !c.cfg.MultiTenant { - // These resources are only installed in zero-tenant clusters. Multi-tenant clusters don't use the default - // RBAC resources. - globalEnterpriseObjects = append(globalEnterpriseObjects, - c.tigeraUserClusterRole(), - c.tigeraNetworkAdminClusterRole(), - ) - } - - if c.cfg.ManagementCluster != nil { - globalEnterpriseObjects = append(globalEnterpriseObjects, c.managedClusterWatchClusterRole()) - if c.cfg.MultiTenant { - // Multi-tenant management cluster API servers need access to per-tenant CA secrets in order to sign - // per-tenant guardian certificates when creating ManagedClusters. - globalEnterpriseObjects = append(globalEnterpriseObjects, c.multiTenantSecretsRBAC()...) - // Multi-tenant management cluster components impersonate the single-tenant canonical service account - // in order to retrieve informations from the managed cluster. A cluster role will be created and each - // component will create a role binding in the tenant namespace - globalEnterpriseObjects = append(globalEnterpriseObjects, c.multiTenantManagedClusterAccessClusterRoles()...) - } else { - globalEnterpriseObjects = append(globalEnterpriseObjects, c.secretsRBAC()...) - } - } else { - // If we're not a management cluster, the API server doesn't need permissions to access secrets. - objsToDelete = append(objsToDelete, c.multiTenantSecretsRBAC()...) - objsToDelete = append(objsToDelete, c.secretsRBAC()...) - objsToDelete = append(objsToDelete, c.multiTenantManagedClusterAccessClusterRoles()...) - objsToDelete = append(objsToDelete, c.managedClusterWatchClusterRole()) - } - - // Namespaced enterprise-only objects. - namespacedEnterpriseObjects := []client.Object{} - - if c.cfg.TrustedBundle != nil { - namespacedEnterpriseObjects = append(namespacedEnterpriseObjects, c.cfg.TrustedBundle.ConfigMap(QueryserverNamespace)) - } + // The sidecar mutating webhook is driven by ApplicationLayer configuration, not by variant. if c.cfg.IsSidecarInjectionEnabled() { - namespacedEnterpriseObjects = append(namespacedEnterpriseObjects, c.sidecarMutatingWebhookConfig()) + namespacedObjects = append(namespacedObjects, c.sidecarMutatingWebhookConfig()) } else { objsToDelete = append(objsToDelete, &admregv1.MutatingWebhookConfiguration{ObjectMeta: metav1.ObjectMeta{Name: common.SidecarMutatingWebhookConfigName}}) } - if c.cfg.ManagementClusterConnection != nil { - namespacedEnterpriseObjects = append(namespacedEnterpriseObjects, - c.externalLinseedRoleBinding(), - ) - } - - // Compile the final arrays based on the variant. - if c.cfg.Installation.Variant.IsEnterprise() { - // Create any enterprise specific objects. - globalObjects = append(globalObjects, globalEnterpriseObjects...) - namespacedObjects = append(namespacedObjects, namespacedEnterpriseObjects...) - - // Clean up cluster-scoped resources that were created with the 'tigera' prefix. - // The apiserver now uses consistent resource names with 'calico' prefix across both EE and OSS variants. - objsToDelete = append(objsToDelete, c.deprecatedResources()...) - } else { - // Explicitly delete any global enterprise objects. - // Namespaced objects will be handled by namespace deletion. - objsToDelete = append(objsToDelete, globalEnterpriseObjects...) - } // Clean up deprecated k8s NetworkPolicy, regardless of variant, // avoiding leftovers in the case of switching between variants. @@ -523,9 +473,9 @@ func calicoSystemAPIServerPolicy(cfg *APIServerConfiguration) *v3.NetworkPolicy Action: v3.Pass, }) - apiServerContainerPort := getContainerPort(cfg, APIServerContainerName).ContainerPort - queryServerContainerPort := getContainerPort(cfg, TigeraAPIServerQueryServerContainerName).ContainerPort - l7AdmCtrlContainerPort := getContainerPort(cfg, L7AdmissionControllerContainerName).ContainerPort + apiServerContainerPort := GetContainerPort(cfg, APIServerContainerName).ContainerPort + queryServerContainerPort := GetContainerPort(cfg, TigeraAPIServerQueryServerContainerName).ContainerPort + l7AdmCtrlContainerPort := GetContainerPort(cfg, L7AdmissionControllerContainerName).ContainerPort // The ports Calico Enterprise API Server and Calico Enterprise Query Server are configured to listen on. ingressPorts := networkpolicy.Ports(443, uint16(apiServerContainerPort), uint16(queryServerContainerPort), 10443) @@ -768,17 +718,6 @@ func (c *apiServerComponent) authClusterRole() client.Object { } } -// multiTenantSecretsRBAC provides the tigera API server with the ability to read secrets on the cluster. -// This is needed in multi-tenant management clusters only, in order to read tenant secrets for signing managed cluster certificates. -func (c *apiServerComponent) multiTenantSecretsRBAC() []client.Object { - return TunnelSecretRBAC(APIServerSecretsRBACName, APIServerServiceAccountName, c.cfg.ManagementCluster, true) -} - -// secretsRBAC provides the tigera API server with the ability to read secrets from the API server's namespace. -func (c *apiServerComponent) secretsRBAC() []client.Object { - return TunnelSecretRBAC(APIServerSecretsRBACName, APIServerServiceAccountName, c.cfg.ManagementCluster, false) -} - // authClusterRoleBinding returns a clusterrolebinding to create, and a clusterrolebinding to delete. // // Both Calico and Calico Enterprise, with different names. @@ -854,7 +793,7 @@ func (c *apiServerComponent) webhookReaderClusterRoleBinding() client.Object { } } -func getContainerPort(cfg *APIServerConfiguration, containerName ContainerName) *operatorv1.APIServerDeploymentContainerPort { +func GetContainerPort(cfg *APIServerConfiguration, containerName ContainerName) *operatorv1.APIServerDeploymentContainerPort { // Try to get the override port if cfg != nil && cfg.APIServer != nil && @@ -889,11 +828,11 @@ func getContainerPort(cfg *APIServerConfiguration, containerName ContainerName) return nil } -// apiServerService creates a service backed by the API server and - for enterprise - query server. +// apiServerService creates a service backed by the API server. A variant modifier may add +// additional ports (e.g. the query server port). func (c *apiServerComponent) apiServerService() *corev1.Service { - apiServerTargetPort := getContainerPort(c.cfg, APIServerContainerName) - queryServerTargetPort := getContainerPort(c.cfg, TigeraAPIServerQueryServerContainerName) - l7AdmissionControllerTargetPort := getContainerPort(c.cfg, L7AdmissionControllerContainerName) + apiServerTargetPort := GetContainerPort(c.cfg, APIServerContainerName) + l7AdmissionControllerTargetPort := GetContainerPort(c.cfg, L7AdmissionControllerContainerName) s := &corev1.Service{ TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, @@ -917,18 +856,6 @@ func (c *apiServerComponent) apiServerService() *corev1.Service { }, } - if c.cfg.Installation.Variant.IsEnterprise() { - // Add port for queryserver if enterprise. - s.Spec.Ports = append(s.Spec.Ports, - corev1.ServicePort{ - Name: QueryServerPortName, - Port: QueryServerPort, - Protocol: corev1.ProtocolTCP, - TargetPort: intstr.FromInt32(queryServerTargetPort.ContainerPort), - }, - ) - } - if c.cfg.IsSidecarInjectionEnabled() { s.Spec.Ports = append(s.Spec.Ports, corev1.ServicePort{ @@ -967,12 +894,13 @@ func (c *apiServerComponent) apiServerDeployment() *appsv1.Deployment { initContainers = append(initContainers, initContainerAPIServer) } - initContainerQueryServer := c.cfg.QueryServerTLSKeyPairCertificateManagementOnly.InitContainer(APIServerNamespace, c.queryServerContainer().SecurityContext) + initContainerQueryServer := c.cfg.QueryServerTLSKeyPairCertificateManagementOnly.InitContainer(APIServerNamespace, securitycontext.NewNonRootContext()) annotations[c.cfg.QueryServerTLSKeyPairCertificateManagementOnly.HashAnnotationKey()] = c.cfg.QueryServerTLSKeyPairCertificateManagementOnly.HashAnnotationValue() initContainers = append(initContainers, initContainerQueryServer) } - // Determine which containers to run. + // Determine which containers to run. A variant modifier may add additional + // containers (e.g. the query server). containers := []corev1.Container{} if c.cfg.RequiresAggregationServer { containers = append(containers, c.apiServerContainer()) @@ -980,9 +908,6 @@ func (c *apiServerComponent) apiServerDeployment() *appsv1.Deployment { if c.cfg.IsSidecarInjectionEnabled() { containers = append(containers, c.l7AdmissionControllerContainer()) } - if c.cfg.Installation.Variant.IsEnterprise() { - containers = append(containers, c.queryServerContainer()) - } d := &appsv1.Deployment{ TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}, @@ -1027,15 +952,6 @@ func (c *apiServerComponent) apiServerDeployment() *appsv1.Deployment { d.Spec.Template.Spec.Affinity = podaffinity.NewPodAntiAffinity(APIServerName, []string{APIServerNamespace, "tigera-system", "calico-apiserver"}) } - if c.cfg.Installation.Variant.IsEnterprise() { - if c.cfg.TrustedBundle != nil { - trustedBundleHashAnnotations := c.cfg.TrustedBundle.HashAnnotations() - for k, v := range trustedBundleHashAnnotations { - d.Spec.Template.Annotations[k] = v - } - } - } - if overrides := c.cfg.APIServer.APIServerDeployment; overrides != nil { rcomp.ApplyDeploymentOverrides(d, overrides) } @@ -1046,7 +962,7 @@ func (c *apiServerComponent) apiServerDeployment() *appsv1.Deployment { // apiServer creates a MutatingWebhookConfiguration for sidecars. func (c *apiServerComponent) sidecarMutatingWebhookConfig() *admregv1.MutatingWebhookConfiguration { var cacert []byte - svcPort := getContainerPort(c.cfg, L7AdmissionControllerContainerName).ContainerPort + svcPort := GetContainerPort(c.cfg, L7AdmissionControllerContainerName).ContainerPort svcpath := "/sidecar-webhook" svcref := admregv1.ServiceReference{ @@ -1103,10 +1019,16 @@ func (c *apiServerComponent) sidecarMutatingWebhookConfig() *admregv1.MutatingWe } func (c *apiServerComponent) hostNetwork() bool { - if c.cfg.ForceHostNetwork { + return HostNetwork(c.cfg) +} + +// HostNetwork reports whether the API server deployment runs on the host network, +// accounting for both the forced setting and the provider-driven requirement. +func HostNetwork(cfg *APIServerConfiguration) bool { + if cfg.ForceHostNetwork { return true } - return HostNetworkRequired(c.cfg.Installation) + return HostNetworkRequired(cfg.Installation) } func HostNetworkRequired(installation *operatorv1.InstallationSpec) bool { @@ -1126,12 +1048,6 @@ func (c *apiServerComponent) apiServerContainer() corev1.Container { volumeMounts := []corev1.VolumeMount{ c.cfg.TLSKeyPair.VolumeMount(c.SupportedOSType()), } - if c.cfg.Installation.Variant.IsEnterprise() { - volumeMounts = append(volumeMounts, - corev1.VolumeMount{Name: auditLogsVolumeName, MountPath: "/var/log/calico/audit"}, - corev1.VolumeMount{Name: auditPolicyVolumeName, MountPath: "/etc/tigera/audit"}, - ) - } env := []corev1.EnvVar{ {Name: "DATASTORE_TYPE", Value: "kubernetes"}, @@ -1161,7 +1077,7 @@ func (c *apiServerComponent) apiServerContainer() corev1.Container { env = append(env, corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: c.cfg.Installation.CalicoNetwork.MultiInterfaceMode.Value()}) } - apiServerTargetPort := getContainerPort(c.cfg, APIServerContainerName).ContainerPort + apiServerTargetPort := GetContainerPort(c.cfg, APIServerContainerName).ContainerPort apiServer := corev1.Container{ Name: string(APIServerContainerName), @@ -1183,19 +1099,13 @@ func (c *apiServerComponent) apiServerContainer() corev1.Container { PeriodSeconds: 60, }, } - // In case of OpenShift, apiserver needs privileged access to write audit logs to host path volume. - // Audit logs are owned by root on hosts so we need to be root user and group. Audit logs are supported only in Enterprise version. - if c.cfg.Installation.Variant.IsEnterprise() { - apiServer.SecurityContext = securitycontext.NewRootContext(c.cfg.OpenShift) - } else { - apiServer.SecurityContext = securitycontext.NewNonRootContext() - } + apiServer.SecurityContext = securitycontext.NewNonRootContext() return apiServer } func (c *apiServerComponent) startUpArgs() []string { - apiServerTargetPort := getContainerPort(c.cfg, APIServerContainerName).ContainerPort + apiServerTargetPort := GetContainerPort(c.cfg, APIServerContainerName).ContainerPort args := []string{ fmt.Sprintf("--secure-port=%d", apiServerTargetPort), @@ -1203,13 +1113,6 @@ func (c *apiServerComponent) startUpArgs() []string { fmt.Sprintf("--tls-cert-file=%s", c.cfg.TLSKeyPair.VolumeMountCertificateFilePath()), } - if c.cfg.Installation.Variant.IsEnterprise() { - args = append(args, - "--audit-policy-file=/etc/tigera/audit/policy.conf", - "--audit-log-path=/var/log/calico/audit/tsee-audit.log", - ) - } - if c.cfg.ManagementCluster != nil { args = append(args, "--enable-managed-clusters-create-api=true") if c.cfg.ManagementCluster.Spec.Address != "" { @@ -1230,122 +1133,6 @@ func (c *apiServerComponent) startUpArgs() []string { return args } -// queryServerContainer creates the query server container. -func (c *apiServerComponent) queryServerContainer() corev1.Container { - queryServerTargetPort := getContainerPort(c.cfg, TigeraAPIServerQueryServerContainerName).ContainerPort - - var tlsSecret certificatemanagement.KeyPairInterface - if c.cfg.QueryServerTLSKeyPairCertificateManagementOnly != nil { - tlsSecret = c.cfg.QueryServerTLSKeyPairCertificateManagementOnly - } else { - tlsSecret = c.cfg.TLSKeyPair - } - env := []corev1.EnvVar{ - {Name: "DATASTORE_TYPE", Value: "kubernetes"}, - {Name: "LISTEN_ADDR", Value: fmt.Sprintf(":%d", queryServerTargetPort)}, - {Name: "TLS_CERT", Value: fmt.Sprintf("/%s/tls.crt", tlsSecret.GetName())}, - {Name: "TLS_KEY", Value: fmt.Sprintf("/%s/tls.key", tlsSecret.GetName())}, - } - if c.cfg.TrustedBundle != nil { - env = append(env, corev1.EnvVar{Name: "TRUSTED_BUNDLE_PATH", Value: c.cfg.TrustedBundle.MountPath()}) - } - - if c.hostNetwork() { - env = append(env, c.cfg.K8SServiceEndpoint.EnvVars()...) - } else { - env = append(env, c.cfg.K8SServiceEndpointPodNetwork.EnvVars()...) - } - - if c.cfg.Installation.CalicoNetwork != nil && c.cfg.Installation.CalicoNetwork.MultiInterfaceMode != nil { - env = append(env, corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: c.cfg.Installation.CalicoNetwork.MultiInterfaceMode.Value()}) - } - - if c.cfg.KeyValidatorConfig != nil { - env = append(env, c.cfg.KeyValidatorConfig.RequiredEnv("")...) - } - - linseedURL := relasticsearch.LinseedEndpoint(c.SupportedOSType(), c.cfg.ClusterDomain, ElasticsearchNamespace, c.cfg.ManagementClusterConnection != nil, false) - env = append(env, - corev1.EnvVar{Name: "LINSEED_URL", Value: linseedURL}, - corev1.EnvVar{Name: "LINSEED_CLIENT_CERT", Value: fmt.Sprintf("/%s/tls.crt", tlsSecret.GetName())}, - corev1.EnvVar{Name: "LINSEED_CLIENT_KEY", Value: fmt.Sprintf("/%s/tls.key", tlsSecret.GetName())}, - ) - if c.cfg.ManagementClusterConnection != nil { - env = append(env, - corev1.EnvVar{Name: "CLUSTER_ID", Value: ""}, - corev1.EnvVar{Name: "LINSEED_TOKEN", Value: GetLinseedTokenPath(true)}, - ) - } - if c.cfg.TrustedBundle != nil { - env = append(env, corev1.EnvVar{Name: "LINSEED_CA", Value: c.cfg.TrustedBundle.MountPath()}) - } - - // set LogLEVEL for queryserver container - if logging := c.cfg.APIServer.Logging; logging != nil && - logging.QueryServerLogging != nil && logging.QueryServerLogging.LogSeverity != nil { - env = append(env, - corev1.EnvVar{Name: "LOGLEVEL", Value: strings.ToLower(string(*logging.QueryServerLogging.LogSeverity))}) - } else { - // set default LOGLEVEL to info when not set by the user - env = append(env, corev1.EnvVar{Name: "LOGLEVEL", Value: "info"}) - } - - volumeMounts := []corev1.VolumeMount{ - tlsSecret.VolumeMount(c.SupportedOSType()), - } - if c.cfg.TrustedBundle != nil { - volumeMounts = append(volumeMounts, c.cfg.TrustedBundle.VolumeMounts(c.SupportedOSType())...) - } - if c.cfg.ManagementClusterConnection != nil { - volumeMounts = append(volumeMounts, corev1.VolumeMount{ - Name: LinseedTokenVolumeName, - MountPath: LinseedVolumeMountPath, - }) - } - - container := corev1.Container{ - Name: string(TigeraAPIServerQueryServerContainerName), - Image: c.calicoImage, - Command: []string{components.CalicoBinaryPath, "component", "queryserver"}, - Env: env, - LivenessProbe: &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{ - Path: "/version", - Port: intstr.FromInt32(queryServerTargetPort), - Scheme: corev1.URISchemeHTTPS, - }, - }, - InitialDelaySeconds: 90, - }, - SecurityContext: securitycontext.NewNonRootContext(), - VolumeMounts: volumeMounts, - } - return container -} - -func (c *apiServerComponent) externalLinseedRoleBinding() *rbacv1.RoleBinding { - return &rbacv1.RoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: "tigera-linseed", - Namespace: APIServerNamespace, - }, - RoleRef: rbacv1.RoleRef{ - APIGroup: "rbac.authorization.k8s.io", - Kind: "ClusterRole", - Name: TigeraLinseedSecretsClusterRole, - }, - Subjects: []rbacv1.Subject{ - { - Kind: "ServiceAccount", - Name: GuardianServiceAccountName, - Namespace: GuardianNamespace, - }, - }, - } -} - // apiServerVolumes creates the volumes used by the API server deployment. func (c *apiServerComponent) apiServerVolumes() []corev1.Volume { volumes := []corev1.Volume{ @@ -1355,40 +1142,6 @@ func (c *apiServerComponent) apiServerVolumes() []corev1.Volume { volumes = append(volumes, c.cfg.QueryServerTLSKeyPairCertificateManagementOnly.Volume()) } - if c.cfg.Installation.Variant.IsEnterprise() && c.cfg.RequiresAggregationServer { - // Only include these volumes if we're running the aggregation API server, since audit logging is done through the - // main API server otherwise. - volumes = append(volumes, - corev1.Volume{ - Name: auditLogsVolumeName, - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/var/log/calico/audit", - Type: ptr.To(corev1.HostPathDirectoryOrCreate), - }, - }, - }, - corev1.Volume{ - Name: auditPolicyVolumeName, - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{Name: auditPolicyVolumeName}, - Items: []corev1.KeyToPath{ - { - Key: "config", - Path: "policy.conf", - }, - }, - }, - }, - }, - ) - } - - if c.cfg.Installation.Variant.IsEnterprise() && c.cfg.TrustedBundle != nil { - volumes = append(volumes, c.cfg.TrustedBundle.Volume()) - } - if c.cfg.ManagementClusterConnection != nil { // Optional: the Secret is delivered over the Guardian tunnel, which can't be // established until calico-apiserver is Ready. @@ -1419,119 +1172,6 @@ func (c *apiServerComponent) tolerations() []corev1.Toleration { return tolerations } -// tigeraAPIServerClusterRole creates a clusterrole that gives permissions to access backing CRDs -// -// Calico Enterprise only -func (c *apiServerComponent) tigeraAPIServerClusterRole() *rbacv1.ClusterRole { - rules := []rbacv1.PolicyRule{ - { - // Read access to Linseed policy activity data for queryserver enrichment. - APIGroups: []string{"linseed.tigera.io"}, - Resources: []string{"policyactivity"}, - Verbs: []string{"get"}, - }, - { - // Calico Enterprise backing storage. - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{ - "alertexceptions", - "bfdconfigurations", - "deeppacketinspections", - "deeppacketinspections/status", - "egressgatewaypolicies", - "externalnetworks", - "globalalerts", - "globalalerts/status", - "globalalerttemplates", - "globalreports", - "globalreports/status", - "globalreporttypes", - "globalthreatfeeds", - "globalthreatfeeds/status", - "licensekeys", - "managedclusters", - "managedclusters/status", - "networks", - "packetcaptures", - "packetcaptures/status", - "policyrecommendationscopes", - "policyrecommendationscopes/status", - "remoteclusterconfigurations", - "securityeventwebhooks", - "securityeventwebhooks/status", - "uisettings", - "uisettingsgroups", - }, - Verbs: []string{ - "get", - "list", - "watch", - "create", - "update", - "delete", - "patch", - }, - }, - { - // The queryserver's RBAC calculator needs to list tiers, - // uisettingsgroups, and managedclusters via the aggregated - // API to evaluate user permissions for the /policies endpoint. - APIGroups: []string{"projectcalico.org"}, - Resources: []string{ - "tiers", - "uisettingsgroups", - "managedclusters", - }, - Verbs: []string{"get", "list", "watch"}, - }, - { - // Required by the AuthorizationReview calculator in queryserver to evaluate - // RBAC permissions for users. - APIGroups: []string{"rbac.authorization.k8s.io"}, - Resources: []string{ - "clusterroles", - "clusterrolebindings", - "roles", - "rolebindings", - }, - Verbs: []string{"get", "list", "watch"}, - }, - } - - return &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: APIServerName, - }, - Rules: rules, - } -} - -// tigeraAPIServerClusterRoleBinding creates a clusterrolebinding that applies tigeraAPIServerClusterRole to -// the calico-apiserver service account. -// -// Calico Enterprise only -func (c *apiServerComponent) tigeraAPIServerClusterRoleBinding() *rbacv1.ClusterRoleBinding { - return &rbacv1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: APIServerName, - }, - Subjects: []rbacv1.Subject{ - { - Kind: "ServiceAccount", - Name: APIServerServiceAccountName, - Namespace: APIServerNamespace, - }, - }, - RoleRef: rbacv1.RoleRef{ - Kind: "ClusterRole", - Name: APIServerName, - APIGroup: "rbac.authorization.k8s.io", - }, - } -} - // tierGetterClusterRole creates a clusterrole that gives permissions to get tiers. func (c *apiServerComponent) tierGetterClusterRole() *rbacv1.ClusterRole { return &rbacv1.ClusterRole{ @@ -1575,699 +1215,86 @@ func (c *apiServerComponent) kubeControllerMgrTierGetterClusterRoleBinding() *rb } } -// uiSettingsGroupGetterClusterRole creates a clusterrole that gives permissions to get uisettingsgroups. -// -// Calico Enterprise only -func (c *apiServerComponent) uiSettingsGroupGetterClusterRole() *rbacv1.ClusterRole { +// calicoPolicyPassthruClusterRole creates a clusterrole that is used to control the RBAC +// mechanism for Calico tiered policy. +func (c *apiServerComponent) calicoPolicyPassthruClusterRole() *rbacv1.ClusterRole { + resources := []string{"networkpolicies", "globalnetworkpolicies"} + return &rbacv1.ClusterRole{ TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: "calico-uisettingsgroup-getter", + Name: "calico-tiered-policy-passthrough", }, + // If tiered policy is enabled we allow all authenticated users to access the main tier resource, instead + // restricting access using the tier.xxx resource type. Kubernetes NetworkPolicy and + // StagedKubernetesNetworkPolicy objects are handled using normal (non-tiered) RBAC. Rules: []rbacv1.PolicyRule{ { APIGroups: []string{"projectcalico.org"}, - Resources: []string{ - "uisettingsgroups", - }, - Verbs: []string{"get"}, + Resources: resources, + Verbs: allVerbs, }, }, } } -// kubeControllerManagerUISettingsGroupGetterClusterRoleBinding creates a rolebinding that allows the k8s kube-controller -// manager to get uisettingsgroups. -// -// In k8s 1.15+, cascading resource deletions (for instance pods for a replicaset) failed due to k8s kube-controller -// not having permissions to get tiers. UISettings and UISettingsGroups RBAC works in a similar way to tiered policy -// and so we need similar RBAC for UISettingsGroups. -// -// Calico Enterprise only -func (c *apiServerComponent) kubeControllerManagerUISettingsGroupGetterClusterRoleBinding() *rbacv1.ClusterRoleBinding { +// calicoPolicyPassthruClusterRolebinding creates a clusterrolebinding that applies calicoPolicyPassthruClusterRole to all users. +func (c *apiServerComponent) calicoPolicyPassthruClusterRolebinding() *rbacv1.ClusterRoleBinding { return &rbacv1.ClusterRoleBinding{ TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: "calico-uisettingsgroup-getter", - }, - RoleRef: rbacv1.RoleRef{ - Kind: "ClusterRole", - Name: "calico-uisettingsgroup-getter", - APIGroup: "rbac.authorization.k8s.io", + Name: "calico-tiered-policy-passthrough", }, Subjects: []rbacv1.Subject{ { - Kind: "User", - Name: "system:kube-controller-manager", + Kind: "Group", + Name: "system:authenticated", APIGroup: "rbac.authorization.k8s.io", }, }, + RoleRef: rbacv1.RoleRef{ + Kind: "ClusterRole", + Name: "calico-tiered-policy-passthrough", + APIGroup: "rbac.authorization.k8s.io", + }, } } -// tigeraUserClusterRole returns a cluster role for a default Calico Enterprise user. -// -// Calico Enterprise only -func (c *apiServerComponent) tigeraUserClusterRole() *rbacv1.ClusterRole { - rules := []rbacv1.PolicyRule{ - // List requests that the Tigera manager needs. - { - APIGroups: []string{ - "projectcalico.org", - "networking.k8s.io", - "extensions", - "", - }, - // Use both the networkpolicies and tier.networkpolicies resource types to ensure identical behavior - // irrespective of the Calico RBAC scheme (see the ClusterRole "calico-tiered-policy-passthrough" for - // more details). Similar for all tiered policy resource types. - Resources: []string{ - "tiers", - "networkpolicies", - "tier.networkpolicies", - "globalnetworkpolicies", - "tier.globalnetworkpolicies", - "namespaces", - "globalnetworksets", - "networksets", - "managedclusters", - "stagedglobalnetworkpolicies", - "tier.stagedglobalnetworkpolicies", - "stagednetworkpolicies", - "tier.stagednetworkpolicies", - "stagedkubernetesnetworkpolicies", - "policyrecommendationscopes", - }, - Verbs: []string{"watch", "list"}, +func (c *apiServerComponent) getDeprecatedResources() []client.Object { + var renamedRscList []client.Object + + // renamed clusterrole tigera-crds to tigera-apiserver + renamedRscList = append(renamedRscList, &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "tigera-crds", }, - { - APIGroups: []string{"policy.networking.k8s.io"}, - Resources: []string{ - "clusternetworkpolicies", - "adminnetworkpolicies", - "baselineadminnetworkpolicies", - }, - Verbs: []string{"watch", "list"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"packetcaptures/files"}, - Verbs: []string{"get"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"packetcaptures"}, - Verbs: []string{"get", "list", "watch"}, - }, - // Additional "list" requests required to view flows. - { - APIGroups: []string{""}, - Resources: []string{"pods"}, - Verbs: []string{"list"}, - }, - // Additional "list" requests required to view serviceaccount labels. - { - APIGroups: []string{""}, - Resources: []string{"serviceaccounts"}, - Verbs: []string{"list"}, - }, - // Access for WAF API to read in coreruleset configmap - { - APIGroups: []string{""}, - Resources: []string{"configmaps"}, - ResourceNames: []string{"coreruleset-default"}, - Verbs: []string{"get"}, - }, - // Access to statistics. - { - APIGroups: []string{""}, - Resources: []string{"services/proxy"}, - ResourceNames: []string{ - "https:calico-api:8080", "calico-node-prometheus:9090", - }, - Verbs: []string{"get", "create"}, - }, - // Access to policies in all tiers - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"tiers"}, - Verbs: []string{"get"}, - }, - // List and download the reports in the Tigera Secure manager. - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"globalreports"}, - Verbs: []string{"get", "list"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"globalreporttypes"}, - Verbs: []string{"get"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"clusterinformations"}, - Verbs: []string{"get", "list"}, - }, - // Access to hostendpoints from the UI ServiceGraph. - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"hostendpoints"}, - Verbs: []string{"get", "list"}, - }, - // List and view the threat defense configuration - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{ - "alertexceptions", - "globalalerts", - "globalalerts/status", - "globalalerttemplates", - "globalthreatfeeds", - "globalthreatfeeds/status", - "securityeventwebhooks", - }, - Verbs: []string{"get", "watch", "list"}, - }, - // User can: - // - read UISettings in the cluster-settings group - // - read and write UISettings in the user-settings group - // Default settings group and settings are created in manager.go. - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"uisettingsgroups"}, - Verbs: []string{"get"}, - ResourceNames: []string{"cluster-settings", "user-settings"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"uisettingsgroups/data"}, - Verbs: []string{"get", "list", "watch"}, - ResourceNames: []string{"cluster-settings"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"uisettingsgroups/data"}, - Verbs: []string{"*"}, - ResourceNames: []string{"user-settings"}, - }, - // Allow the user to read applicationlayers to detect if WAF is enabled/disabled. - { - APIGroups: []string{"operator.tigera.io"}, - Resources: []string{"applicationlayers", "packetcaptureapis", "compliances", "intrusiondetections"}, - Verbs: []string{"get"}, - }, - { - APIGroups: []string{"apps"}, - Resources: []string{"deployments"}, - Verbs: []string{"get", "list", "watch"}, - }, - // Allow the user to read services to view WAF configuration. - { - APIGroups: []string{""}, - Resources: []string{"services"}, - Verbs: []string{"get", "list", "watch"}, - }, - // Allow the user to read felixconfigurations to detect if wireguard and/or other features are enabled. - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"felixconfigurations"}, - Verbs: []string{"get", "list"}, - }, - // Allow the user to only view securityeventwebhooks. - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"securityeventwebhooks"}, - Verbs: []string{"get", "list"}, - }, - } - - // Privileges for lma.tigera.io have no effect on managed clusters. - if c.cfg.ManagementClusterConnection == nil { - // Access to flow logs, audit logs, and statistics. - // Access to log into Kibana for oidc users. - rules = append(rules, rbacv1.PolicyRule{ - APIGroups: []string{"lma.tigera.io"}, - Resources: []string{"*"}, - ResourceNames: []string{ - "flows", "audit*", "l7", "events", "dns", "waf", "kibana_login", "recommendations", - }, - Verbs: []string{"get"}, - }) - } - - return &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: "tigera-ui-user", - }, - Rules: rules, - } -} - -// tigeraNetworkAdminClusterRole returns a cluster role for a Tigera Secure manager network admin. -// -// Calico Enterprise only -func (c *apiServerComponent) tigeraNetworkAdminClusterRole() *rbacv1.ClusterRole { - rules := []rbacv1.PolicyRule{ - // Full access to all network policies - { - APIGroups: []string{ - "projectcalico.org", - "networking.k8s.io", - "extensions", - }, - // Use both the networkpolicies and tier.networkpolicies resource types to ensure identical behavior - // irrespective of the Calico RBAC scheme (see the ClusterRole "calico-tiered-policy-passthrough" for - // more details). Similar for all tiered policy resource types. - Resources: []string{ - "tiers", - "networkpolicies", - "tier.networkpolicies", - "globalnetworkpolicies", - "tier.globalnetworkpolicies", - "stagedglobalnetworkpolicies", - "tier.stagedglobalnetworkpolicies", - "stagednetworkpolicies", - "tier.stagednetworkpolicies", - "stagedkubernetesnetworkpolicies", - "globalnetworksets", - "networksets", - "managedclusters", - "packetcaptures", - "policyrecommendationscopes", - }, - Verbs: []string{"create", "update", "delete", "patch", "get", "watch", "list"}, - }, - { - APIGroups: []string{ - "policy.networking.k8s.io", - }, - Resources: []string{ - "clusternetworkpolicies", - "adminnetworkpolicies", - "baselineadminnetworkpolicies", - }, - Verbs: []string{"create", "update", "delete", "patch", "get", "watch", "list"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"packetcaptures/files"}, - Verbs: []string{"get", "delete"}, - }, - // Additional "list" requests that the Tigera Secure manager needs - { - APIGroups: []string{""}, - Resources: []string{"namespaces"}, - Verbs: []string{"watch", "list"}, - }, - // Additional "list" requests required to view flows. - { - APIGroups: []string{""}, - Resources: []string{"pods"}, - Verbs: []string{"list"}, - }, - // Additional "list" requests required to view serviceaccount labels. - { - APIGroups: []string{""}, - Resources: []string{"serviceaccounts"}, - Verbs: []string{"list"}, - }, - // Access for WAF API to read in coreruleset configmap - { - APIGroups: []string{""}, - Resources: []string{"configmaps"}, - ResourceNames: []string{"coreruleset-default"}, - Verbs: []string{"get"}, - }, - // Access to statistics. - { - APIGroups: []string{""}, - Resources: []string{"services/proxy"}, - ResourceNames: []string{ - "https:calico-api:8080", "calico-node-prometheus:9090", - }, - Verbs: []string{"get", "create"}, - }, - // Manage globalreport configuration, view report generation status, and list reports in the Tigera Secure manager. - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"globalreports"}, - Verbs: []string{"*"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"globalreports/status"}, - Verbs: []string{"get", "list", "watch"}, - }, - // List and download the reports in the Tigera Secure manager. - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"globalreporttypes"}, - Verbs: []string{"get"}, - }, - // Access to cluster information containing Calico and EE versions from the UI. - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"clusterinformations"}, - Verbs: []string{"get", "list"}, - }, - // Access to hostendpoints from the UI ServiceGraph. - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"hostendpoints"}, - Verbs: []string{"get", "list"}, - }, - // Manage the threat defense configuration - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{ - "alertexceptions", - "globalalerts", - "globalalerts/status", - "globalalerttemplates", - "globalthreatfeeds", - "globalthreatfeeds/status", - "securityeventwebhooks", - }, - Verbs: []string{"create", "update", "delete", "patch", "get", "watch", "list"}, - }, - // User can: - // - read and write UISettings in the cluster-settings group, and rename the group - // - read and write UISettings in the user-settings group, and rename the group - // Default settings group and settings are created in manager.go. - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"uisettingsgroups"}, - Verbs: []string{"get", "patch", "update"}, - ResourceNames: []string{"cluster-settings", "user-settings"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"uisettingsgroups/data"}, - Verbs: []string{"*"}, - ResourceNames: []string{"cluster-settings", "user-settings"}, - }, - // Allow the user to read and write applicationlayers to enable/disable WAF. - { - APIGroups: []string{"operator.tigera.io"}, - Resources: []string{"applicationlayers", "packetcaptureapis", "compliances", "intrusiondetections"}, - Verbs: []string{"get", "update", "patch", "create", "delete"}, - }, - // Allow the user to read deployments to view WAF configuration. - { - APIGroups: []string{"apps"}, - Resources: []string{"deployments"}, - Verbs: []string{"get", "list", "watch", "patch"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"services"}, - Verbs: []string{"get", "list", "watch", "patch"}, - }, - // Allow the user to read felixconfigurations to detect if wireguard and/or other features are enabled. - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"felixconfigurations"}, - Verbs: []string{"get", "list"}, - }, - // Allow the user to perform CRUD operations on securityeventwebhooks. - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"securityeventwebhooks"}, - Verbs: []string{"get", "list", "update", "patch", "create", "delete"}, - }, - // Allow the user to create secrets. - { - APIGroups: []string{""}, - Resources: []string{ - "secrets", - }, - Verbs: []string{"create"}, - }, - // Allow the user to patch webhooks-secret secret. - { - APIGroups: []string{""}, - Resources: []string{ - "secrets", - }, - ResourceNames: []string{ - "webhooks-secret", - }, - Verbs: []string{"patch"}, - }, - } - - // Privileges for lma.tigera.io have no effect on managed clusters. - if c.cfg.ManagementClusterConnection == nil { - // Access to flow logs, audit logs, and statistics. - // Elasticsearch superuser access once logged into Kibana. - rules = append(rules, rbacv1.PolicyRule{ - APIGroups: []string{"lma.tigera.io"}, - Resources: []string{"*"}, - ResourceNames: []string{ - "flows", "audit*", "l7", "events", "dns", "waf", "kibana_login", "elasticsearch_superuser", "recommendations", - }, - Verbs: []string{"get"}, - }) - } - - // In v3 CRD / webhooks mode there is no aggregated apiserver, and the - // calico-uisettings-passthrough ClusterRole that normally grants the broad - // uisettings permission isn't deployed. Grant write verbs here so the - // calico-webhooks UISettings handler (which narrows access via a SAR on - // uisettingsgroups/data) gets invoked instead of being short-circuited by - // kube-apiserver RBAC. - if !c.cfg.RequiresAggregationServer { - rules = append(rules, rbacv1.PolicyRule{ - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"uisettings"}, - Verbs: []string{"create", "update", "delete", "patch"}, - }) - } - - return &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: "tigera-network-admin", - }, - Rules: rules, - } -} - -// calicoPolicyPassthruClusterRole creates a clusterrole that is used to control the RBAC -// mechanism for Calico tiered policy. -func (c *apiServerComponent) calicoPolicyPassthruClusterRole() *rbacv1.ClusterRole { - resources := []string{"networkpolicies", "globalnetworkpolicies"} - - // Append additional resources for enterprise Variant. - if c.cfg.Installation.Variant.IsEnterprise() { - resources = append(resources, "stagednetworkpolicies", "stagedglobalnetworkpolicies") - } - - return &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: "calico-tiered-policy-passthrough", - }, - // If tiered policy is enabled we allow all authenticated users to access the main tier resource, instead - // restricting access using the tier.xxx resource type. Kubernetes NetworkPolicy and - // StagedKubernetesNetworkPolicy objects are handled using normal (non-tiered) RBAC. - Rules: []rbacv1.PolicyRule{ - { - APIGroups: []string{"projectcalico.org"}, - Resources: resources, - Verbs: allVerbs, - }, - }, - } -} + }) -// calicoPolicyPassthruClusterRolebinding creates a clusterrolebinding that applies calicoPolicyPassthruClusterRole to all users. -func (c *apiServerComponent) calicoPolicyPassthruClusterRolebinding() *rbacv1.ClusterRoleBinding { - return &rbacv1.ClusterRoleBinding{ + // renamed clusterrolebinding tigera-apiserver-access-tigera-crds to tigera-apiserver + renamedRscList = append(renamedRscList, &rbacv1.ClusterRoleBinding{ TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: "calico-tiered-policy-passthrough", - }, - Subjects: []rbacv1.Subject{ - { - Kind: "Group", - Name: "system:authenticated", - APIGroup: "rbac.authorization.k8s.io", - }, - }, - RoleRef: rbacv1.RoleRef{ - Kind: "ClusterRole", - Name: "calico-tiered-policy-passthrough", - APIGroup: "rbac.authorization.k8s.io", - }, - } -} - -// uiSettingsPassthruClusterRole creates a clusterrole that is used to control the RBAC mechanism for Tigera UI Settings. -// RBAC for these is handled within the Tigera API Server which checks uisettingsgroups/data permissions for the user. -// -// Calico Enterprise only -func (c *apiServerComponent) uiSettingsPassthruClusterRole() *rbacv1.ClusterRole { - return &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: "calico-uisettings-passthrough", - }, - Rules: []rbacv1.PolicyRule{ - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"uisettings"}, - Verbs: []string{"*"}, - }, + Name: "tigera-apiserver-access-tigera-crds", }, - } -} + }) -// uiSettingsPassthruClusterRolebinding creates a clusterrolebinding that applies uiSettingsPassthruClusterRole to all -// users. -// -// Calico Enterprise only. -func (c *apiServerComponent) uiSettingsPassthruClusterRolebinding() *rbacv1.ClusterRoleBinding { - return &rbacv1.ClusterRoleBinding{ + // Renamed ClusterRoleBinding tigera-tier-getter to calico-tier-getter since Tier is available in OSS. + // Deleting an object that was never created (e.g. in a fresh OSS install) is a no-op. + renamedRscList = append(renamedRscList, &rbacv1.ClusterRoleBinding{ TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: "calico-uisettings-passthrough", - }, - Subjects: []rbacv1.Subject{ - { - Kind: "Group", - Name: "system:authenticated", - APIGroup: "rbac.authorization.k8s.io", - }, - }, - RoleRef: rbacv1.RoleRef{ - Kind: "ClusterRole", - Name: "calico-uisettings-passthrough", - APIGroup: "rbac.authorization.k8s.io", - }, - } -} - -// auditPolicyConfigMap returns a configmap with contents to configure audit logging for -// projectcalico.org/v3 APIs. -// -// Calico Enterprise only -func (c *apiServerComponent) auditPolicyConfigMap() *corev1.ConfigMap { - const defaultAuditPolicy = `apiVersion: audit.k8s.io/v1 -kind: Policy -rules: -- level: RequestResponse - omitStages: - - RequestReceived - verbs: - - create - - patch - - update - - delete - resources: - - group: projectcalico.org - resources: - - globalnetworkpolicies - - networkpolicies - - stagedglobalnetworkpolicies - - stagednetworkpolicies - - stagedkubernetesnetworkpolicies - - globalnetworksets - - networksets - - tiers - - hostendpoints` - - return &corev1.ConfigMap{ - TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}, - ObjectMeta: metav1.ObjectMeta{ - // This object is for Enterprise only, so pass it explicitly. - Namespace: APIServerNamespace, - Name: auditPolicyVolumeName, - }, - Data: map[string]string{ - "config": defaultAuditPolicy, - }, - } -} - -func (c *apiServerComponent) multiTenantManagedClusterAccessClusterRoles() []client.Object { - var objects []client.Object - objects = append(objects, &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: MultiTenantManagedClustersAccessClusterRoleName}, - Rules: []rbacv1.PolicyRule{ - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"managedclusters"}, - Verbs: []string{ - // The Authentication Proxy in Voltron checks if Enterprise Components (using impersonation headers for - // the service in the canonical namespace) can get a managed clusters before sending the request down the tunnel. - // This ClusterRole will be assigned to each component using a RoleBinding in the canonical or tenant namespace. - "get", - }, - }, + Name: "tigera-tier-getter", }, }) - - return objects -} - -// managedClusterWatchClusterRole creates a ClusterRole for watching the ManagedCluster API -func (c *apiServerComponent) managedClusterWatchClusterRole() client.Object { - return &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: ManagedClustersWatchClusterRoleName}, - Rules: []rbacv1.PolicyRule{ - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"managedclusters"}, - Verbs: []string{ - "get", "list", "watch", - }, - }, - }, - } -} - -func (c *apiServerComponent) getDeprecatedResources() []client.Object { - var renamedRscList []client.Object - - // renamed clusterrole tigera-crds to tigera-apiserver + // Renamed ClusterRole tigera-tier-getter to calico-tier-getter since Tier is available in OSS renamedRscList = append(renamedRscList, &rbacv1.ClusterRole{ TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: "tigera-crds", + Name: "tigera-tier-getter", }, }) - // renamed clusterrolebinding tigera-apiserver-access-tigera-crds to tigera-apiserver - renamedRscList = append(renamedRscList, &rbacv1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: "tigera-apiserver-access-tigera-crds", - }, - }) - - // The following resources were not present in Calico OSS, so there is no need to clean up in OSS. - if c.cfg.Installation.Variant.IsEnterprise() { - // Renamed ClusterRoleBinging tigera-tier-getter to calico-tier-getter since Tier is available in OSS - renamedRscList = append(renamedRscList, &rbacv1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: "tigera-tier-getter", - }, - }) - // Renamed ClusterRole tigera-tier-getter to calico-tier-getter since Tier is available in OSS - renamedRscList = append(renamedRscList, &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: "tigera-tier-getter", - }, - }) - } - renamedRscList = append(renamedRscList, &corev1.Namespace{ TypeMeta: metav1.TypeMeta{Kind: "Namespace", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{ @@ -2302,7 +1329,7 @@ func (c *apiServerComponent) l7AdmissionControllerContainer() corev1.Container { c.cfg.TLSKeyPair.VolumeMount(c.SupportedOSType()), } - l7AdmissionControllerTargetPort := getContainerPort(c.cfg, L7AdmissionControllerContainerName).ContainerPort + l7AdmissionControllerTargetPort := GetContainerPort(c.cfg, L7AdmissionControllerContainerName).ContainerPort dataplane := "iptables" if c.cfg.Installation.IsNftables() { @@ -2353,99 +1380,3 @@ func (c *apiServerComponent) l7AdmissionControllerContainer() corev1.Container { return l7AdmssCtrl } - -// deprecatedResources removes legacy cluster-scoped resources created with the 'tigera' prefix (EE-only). -// Moving forward, both EE and OSS variants standardize on the 'calico' prefix for all shared resources. -// TODO to clean up the below deprecated logic with 14 resources in 3.25+ -func (c *apiServerComponent) deprecatedResources() []client.Object { - return []client.Object{ - &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-extension-apiserver-secrets-access"}, - }, - &rbacv1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-extension-apiserver-secrets-access"}, - }, - - // delegateAuthClusterRoleBinding - &rbacv1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-apiserver-delegate-auth"}, - }, - - // authClusterRole - &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-extension-apiserver-auth-access"}, - }, - - // authClusterRoleBinding - &rbacv1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-extension-apiserver-auth-access"}, - }, - // authReaderRoleBinding - need clean up in diff namespace kube-system - &rbacv1.RoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: "tigera-auth-reader", - Namespace: "kube-system", - }, - }, - // webhookReaderClusterRole - &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-webhook-reader"}, - }, - - // webhookReaderClusterRoleBinding - &rbacv1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-apiserver-webhook-reader"}, - }, - - // calico-apiserver CR and CRB - &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-apiserver"}, - }, - &rbacv1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-apiserver"}, - }, - - &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-uisettingsgroup-getter"}, - }, - &rbacv1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-uisettingsgroup-getter"}, - }, - - &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-tiered-policy-passthrough"}, - }, - &rbacv1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-tiered-policy-passthrough"}, - }, - - &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-uisettings-passthrough"}, - }, - &rbacv1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-uisettings-passthrough"}, - }, - - // Clean up legacy secrets in the tigera-operator namespace - &corev1.Secret{ - TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-api-cert", Namespace: common.OperatorNamespace()}, - }, - } -} diff --git a/pkg/render/apiserver_test.go b/pkg/render/apiserver_test.go index 4efa14a95a..87317ccc0d 100644 --- a/pkg/render/apiserver_test.go +++ b/pkg/render/apiserver_test.go @@ -38,6 +38,7 @@ import ( "github.com/tigera/operator/pkg/controller/k8sapi" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" rmeta "github.com/tigera/operator/pkg/render/common/meta" "github.com/tigera/operator/pkg/render/common/networkpolicy" @@ -60,6 +61,21 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) +// apiServerObjects renders the API server component and applies the registered variant +// modifier the way the componentHandler does, so the Enterprise objects (query server, +// audit logging, Enterprise RBAC) and the Calico cleanup deletes are reflected in the +// returned create and delete lists. +func apiServerObjects(c render.Component) ([]client.Object, []client.Object) { + create, del := c.Objects() + rc := extensions.RenderContext{} + if p, ok := c.(render.ExtensionContextProvider); ok { + ec := p.ExtensionContext().(render.APIServerExtensionContext) + rc.Installation = ec.Config.Installation + rc.Component = ec + } + return extensions.ApplyModifiers(render.ComponentNameAPIServer, rc, create, del) +} + var _ = Describe("API server rendering tests (Calico Enterprise)", func() { apiServerPolicy := testutils.GetExpectedPolicyFromFile("./testutils/expected_policies/apiserver.json") apiServerPolicyForOCP := testutils.GetExpectedPolicyFromFile("./testutils/expected_policies/apiserver_ocp.json") @@ -97,6 +113,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { cfg = &render.APIServerConfiguration{ RequiresAggregationServer: true, + RequiresQueryServer: true, K8SServiceEndpoint: k8sapi.ServiceEndpoint{}, Installation: instance, APIServer: apiserver, @@ -149,7 +166,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) // Should render the correct resources. // - 1 namespace @@ -389,7 +406,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) expectedResources := []client.Object{ &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "tigera-ca-bundle", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, @@ -427,7 +444,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { It("should grant the calico-apiserver SA write access to globalreports/status", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) cr := rtest.GetResource(resources, "calico-apiserver", "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) @@ -465,7 +482,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) d, ok := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(ok).To(BeTrue()) @@ -503,7 +520,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { } component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) deploy, ok := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(ok).To(BeTrue()) @@ -535,7 +552,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { component, err := render.APIServer(cfg) Expect(err).NotTo(HaveOccurred()) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) role := rtest.GetResource(resources, "calico-extension-apiserver-auth-access", "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) Expect(role.Rules).To(ContainElement(rbacv1.PolicyRule{ @@ -585,7 +602,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) rtest.ExpectResources(resources, expectedResources) @@ -630,7 +647,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) rtest.ExpectResources(resources, expectedResources) @@ -699,7 +716,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) rtest.ExpectResources(resources, expectedResources) @@ -720,7 +737,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) d := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(d.Spec.Template.Spec.Tolerations).To(ContainElements(append(rmeta.TolerateControlPlane, tol))) }) @@ -759,7 +776,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) rtest.ExpectResources(resources, expectedResources) @@ -790,7 +807,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) deploymentResource := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment") Expect(deploymentResource).ToNot(BeNil()) @@ -803,7 +820,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { cfg.ForceHostNetwork = true component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) d := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(d.Spec.Strategy.Type).To(Equal(appsv1.RecreateDeploymentStrategyType)) }) @@ -814,7 +831,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { cfg.ForceHostNetwork = true component := render.APIServerPolicy(cfg) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) policyName := types.NamespacedName{Name: "calico-system.apiserver-access", Namespace: "calico-system"} policy := testutils.GetCalicoSystemPolicyFromResources(policyName, resources) Expect(policy).ToNot(BeNil()) @@ -836,7 +853,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { cfg.ForceHostNetwork = false component := render.APIServerPolicy(cfg) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) policyName := types.NamespacedName{Name: "calico-system.apiserver-access", Namespace: "calico-system"} policy := testutils.GetCalicoSystemPolicyFromResources(policyName, resources) Expect(policy).ToNot(BeNil()) @@ -860,7 +877,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) deploymentResource := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment") Expect(deploymentResource).ToNot(BeNil()) @@ -877,7 +894,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) deploymentResource := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment") Expect(deploymentResource).ToNot(BeNil()) @@ -892,7 +909,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) expectedResources := []client.Object{ &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "calico-audit-policy", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, @@ -958,7 +975,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) expected := []client.Object{ &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "calico-audit-policy", Namespace: "calico-system"}}, @@ -1018,7 +1035,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) dep := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment") Expect(dep).ToNot(BeNil()) @@ -1035,7 +1052,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) dep := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment") Expect(dep).ToNot(BeNil()) @@ -1057,7 +1074,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { Expect(err).NotTo(HaveOccurred()) component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) expectedResources := []client.Object{ &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "calico-audit-policy", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "tigera-ca-bundle", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, @@ -1102,7 +1119,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { cfg.Installation.ControlPlaneReplicas = ptr.To(int32(1)) component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) deploy, ok := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(ok).To(BeTrue()) @@ -1113,7 +1130,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { cfg.Installation.ControlPlaneReplicas = ptr.To(int32(2)) component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) deploy, ok := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(ok).To(BeTrue()) @@ -1127,7 +1144,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) rb, ok := rtest.GetResource(resources, "tigera-linseed", "calico-system", "rbac.authorization.k8s.io", "v1", "RoleBinding").(*rbacv1.RoleBinding) Expect(ok).To(BeTrue(), "expected tigera-linseed RoleBinding in calico-system") @@ -1179,7 +1196,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { } component := render.APIServerPolicy(cfg) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) policy := testutils.GetCalicoSystemPolicyFromResources(policyName, resources) expectedPolicy := testutils.SelectPolicyByProvider(scenario, apiServerPolicy, apiServerPolicyForOCP) @@ -1325,7 +1342,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) d, ok := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(ok).To(BeTrue()) @@ -1438,7 +1455,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) d, ok := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(ok).To(BeTrue()) // nodeSelectors are merged @@ -1469,7 +1486,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) d, ok := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(ok).To(BeTrue()) Expect(d.Spec.Template.Spec.Tolerations).To(HaveLen(1)) @@ -1483,7 +1500,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { } component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) d := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(d.Spec.Template.Spec.Containers[0].Args).To(ConsistOf([]string{ "--secure-port=5443", @@ -1945,7 +1962,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, deleteResources := component.Objects() + resources, deleteResources := apiServerObjects(component) rtest.ExpectResources(resources, expectedResources) rtest.ExpectResourceInList(deleteResources, "allow-apiserver", "calico-system", "networking.k8s.io", "v1", "NetworkPolicy") @@ -2041,7 +2058,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, deleteResources := component.Objects() + resources, deleteResources := apiServerObjects(component) // Should not include deployment, service, SA, or PDB. Expect(rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment")).To(BeNil()) @@ -2084,7 +2101,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, deleteResources := component.Objects() + resources, deleteResources := apiServerObjects(component) // Should render the correct resources. By("Checking each expected resource is actually rendered") @@ -2121,7 +2138,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) d := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(d.Spec.Template.Spec.NodeSelector).To(HaveLen(1)) Expect(d.Spec.Template.Spec.NodeSelector).To(HaveKeyWithValue("nodeName", "control01")) @@ -2137,7 +2154,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { cfg.Installation.ControlPlaneTolerations = []corev1.Toleration{tol} component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) d := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(d.Spec.Template.Spec.Tolerations).To(ContainElements(append(rmeta.TolerateControlPlane, tol))) }) @@ -2151,7 +2168,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) deploymentResource := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment") Expect(deploymentResource).ToNot(BeNil()) @@ -2164,7 +2181,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { cfg.ForceHostNetwork = true component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) d := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(d.Spec.Strategy.Type).To(Equal(appsv1.RecreateDeploymentStrategyType)) }) @@ -2177,7 +2194,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) deploymentResource := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment") Expect(deploymentResource).ToNot(BeNil()) @@ -2194,7 +2211,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) deploymentResource := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment") Expect(deploymentResource).ToNot(BeNil()) @@ -2209,7 +2226,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) deploy, ok := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(ok).To(BeTrue()) @@ -2222,7 +2239,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) deploy, ok := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(ok).To(BeTrue()) @@ -2236,7 +2253,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - _, _ = component.Objects() + _, _ = apiServerObjects(component) }) It("should render host networked with TKG provider", func() { @@ -2247,7 +2264,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) deploy, ok := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(ok).To(BeTrue()) @@ -2359,7 +2376,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) d, ok := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(ok).To(BeTrue()) @@ -2433,7 +2450,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) d := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) // nodeSelectors are merged Expect(d.Spec.Template.Spec.NodeSelector).To(HaveLen(2)) @@ -2463,7 +2480,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) d := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(d.Spec.Template.Spec.Tolerations).To(HaveLen(1)) Expect(d.Spec.Template.Spec.Tolerations).To(ConsistOf(tol)) @@ -2475,7 +2492,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).NotTo(HaveOccurred(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).NotTo(HaveOccurred()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) d := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(d).NotTo(BeNil()) Expect(d.Spec.Template.Spec.Tolerations).To(ContainElement(corev1.Toleration{ @@ -2503,7 +2520,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { Expect(err).NotTo(HaveOccurred()) // Expect no UISettings / UISettingsGroups to be installed. - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) obj := rtest.GetResource(resources, "tigera-network-admin", "", "rbac.authorization.k8s.io", "v1", "ClusterRole") Expect(obj).To(BeNil()) obj = rtest.GetResource(resources, "tigera-ui-user", "", "rbac.authorization.k8s.io", "v1", "ClusterRole") @@ -2514,7 +2531,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).NotTo(HaveOccurred()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) managedClusterAccessRole := rtest.GetResource(resources, render.MultiTenantManagedClustersAccessClusterRoleName, "", rbacv1.GroupName, "v1", "ClusterRole").(*rbacv1.ClusterRole) expectedManagedClusterAccessRules := []rbacv1.PolicyRule{ @@ -2531,7 +2548,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).NotTo(HaveOccurred()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) managedClusterAccessRole := rtest.GetResource(resources, render.ManagedClustersWatchClusterRoleName, "", rbacv1.GroupName, "v1", "ClusterRole").(*rbacv1.ClusterRole) expectedManagedClusterAccessRules := []rbacv1.PolicyRule{ diff --git a/pkg/render/guardian_test.go b/pkg/render/guardian_test.go index 244e1c6fa5..b28e77f044 100644 --- a/pkg/render/guardian_test.go +++ b/pkg/render/guardian_test.go @@ -54,7 +54,8 @@ func guardianObjects(cfg *render.GuardianConfiguration) []client.Object { if p, ok := g.(render.ExtensionContextProvider); ok { rc.Component = p.ExtensionContext() } - return extensions.ApplyModifiers(render.GuardianName, rc, objs) + out, _ := extensions.ApplyModifiers(render.GuardianName, rc, objs, nil) + return out } var _ = Describe("Rendering tests", func() { @@ -115,7 +116,7 @@ var _ = Describe("Rendering tests", func() { if p, ok := g.(render.ExtensionContextProvider); ok { rc.Component = p.ExtensionContext() } - resources = extensions.ApplyModifiers(render.GuardianName, rc, resources) + resources, _ = extensions.ApplyModifiers(render.GuardianName, rc, resources, nil) } BeforeEach(func() { @@ -351,7 +352,7 @@ var _ = Describe("Rendering tests", func() { if p, ok := g.(render.ExtensionContextProvider); ok { rc.Component = p.ExtensionContext() } - resources = extensions.ApplyModifiers(render.ComponentNameGuardianPolicy, rc, objs) + resources, _ = extensions.ApplyModifiers(render.ComponentNameGuardianPolicy, rc, objs, nil) } Context("policy rendering based on variant and IncludeEgressNetworkPolicy", func() { diff --git a/pkg/render/node_enterprise_test.go b/pkg/render/node_enterprise_test.go index 9e0d9c493b..4b9b6a2f82 100644 --- a/pkg/render/node_enterprise_test.go +++ b/pkg/render/node_enterprise_test.go @@ -110,7 +110,8 @@ var _ = Describe("node enterprise modifier integration", func() { comp := render.Node(cfg) Expect(comp.ResolveImages(nil)).NotTo(HaveOccurred()) objs, _ := comp.Objects() - return extensions.ApplyModifiers(render.ComponentNameNode, renderCtx, objs) + out, _ := extensions.ApplyModifiers(render.ComponentNameNode, renderCtx, objs, nil) + return out } It("appends the node metrics service to the real render output", func() { @@ -166,7 +167,7 @@ var _ = Describe("node enterprise modifier integration", func() { }) Expect(comp.ResolveImages(nil)).NotTo(HaveOccurred()) objs, _ := comp.Objects() - objs = extensions.ApplyModifiers(render.ComponentNameTypha, renderCtx, objs) + objs, _ = extensions.ApplyModifiers(render.ComponentNameTypha, renderCtx, objs, nil) role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, "calico-typha") Expect(ok).To(BeTrue()) diff --git a/pkg/render/windows_test.go b/pkg/render/windows_test.go index 1eb52c0849..b19063e069 100644 --- a/pkg/render/windows_test.go +++ b/pkg/render/windows_test.go @@ -54,7 +54,8 @@ func renderWindows(cfg *render.WindowsConfiguration) []client.Object { if p, ok := comp.(render.ExtensionContextProvider); ok { rc.Component = p.ExtensionContext() } - return extensions.ApplyModifiers(render.ComponentNameWindows, rc, objs) + out, _ := extensions.ApplyModifiers(render.ComponentNameWindows, rc, objs, nil) + return out } var _ = Describe("Windows rendering tests", func() { From 139aabba3be9d3fdf57c502ac74e4f53d512d668 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Wed, 17 Jun 2026 09:27:16 -0700 Subject: [PATCH 25/38] Address review feedback: cleanups Trim over-commenting on the component handler calls and expand the ones threading a render context to one arg per line. Rename RunSetup to BuildContext, drop the nodePrometheusTLS temp var, and log a BUG on a failed per-component context assertion instead of failing silently. --- .../apiserver/apiserver_controller.go | 13 +++++++----- .../clusterconnection_controller.go | 11 ++++++---- .../installation/core_controller.go | 21 +++++++++---------- .../installation/windows_controller.go | 11 ++++++---- pkg/enterprise/apiserver.go | 13 ++++++++++-- pkg/enterprise/guardian.go | 12 +++++++++-- pkg/enterprise/installation_test.go | 6 +++--- pkg/enterprise/windows.go | 7 ++++++- pkg/extensions/doc.go | 2 +- pkg/extensions/setup.go | 7 ++++--- pkg/extensions/setup_test.go | 10 ++++----- 11 files changed, 72 insertions(+), 41 deletions(-) diff --git a/pkg/controller/apiserver/apiserver_controller.go b/pkg/controller/apiserver/apiserver_controller.go index 0e762c1717..402e784920 100644 --- a/pkg/controller/apiserver/apiserver_controller.go +++ b/pkg/controller/apiserver/apiserver_controller.go @@ -474,11 +474,14 @@ func (r *ReconcileAPIServer) Reconcile(ctx context.Context, request reconcile.Re return reconcile.Result{}, err } - // Create a component handler to manage the rendered component. The render context - // carries the installation so the componentHandler applies the variant's API server - // modifier (query server, audit logging, Enterprise RBAC) to the rendered objects. - handler := utils.NewComponentHandler(log, r.client, r.scheme, instance, - utils.WithRenderContext(extensions.RenderContext{Installation: installationSpec})) + // Create a component handler to manage the rendered component. + handler := utils.NewComponentHandler( + log, + r.client, + r.scheme, + instance, + utils.WithRenderContext(extensions.RenderContext{Installation: installationSpec}), + ) // Render the desired objects from the CRD and create or update them. reqLogger.V(3).Info("rendering components") diff --git a/pkg/controller/clusterconnection/clusterconnection_controller.go b/pkg/controller/clusterconnection/clusterconnection_controller.go index b46ad0c205..d796e01638 100644 --- a/pkg/controller/clusterconnection/clusterconnection_controller.go +++ b/pkg/controller/clusterconnection/clusterconnection_controller.go @@ -444,10 +444,13 @@ func (r *ReconcileConnection) Reconcile(ctx context.Context, request reconcile.R return reconcile.Result{}, err } - // The render context carries the installation so registered modifiers gate on - // variant; the guardian component supplies its own per-component context (the - // impersonation config, OpenShift, and CA bundle path) via ExtensionContext. - ch := utils.NewComponentHandler(log, r.cli, r.scheme, managementClusterConnection, utils.WithRenderContext(extensions.RenderContext{Installation: installationSpec})) + ch := utils.NewComponentHandler( + log, + r.cli, + r.scheme, + managementClusterConnection, + utils.WithRenderContext(extensions.RenderContext{Installation: installationSpec}), + ) guardianCfg := &render.GuardianConfiguration{ URL: managementClusterConnection.Spec.ManagementClusterAddr, PodProxies: r.resolvedPodProxies, diff --git a/pkg/controller/installation/core_controller.go b/pkg/controller/installation/core_controller.go index 65ef9def74..d760aed637 100644 --- a/pkg/controller/installation/core_controller.go +++ b/pkg/controller/installation/core_controller.go @@ -1226,13 +1226,7 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile calicoVersion = components.EnterpriseRelease } - // Run the variant setup to build the render context handed to registered - // modifiers. The core operator has no setup, so it gets just the base - // context; an extension's setup additionally does controller-side work for - // its variant - validating config and creating the node-prometheus - // certificate, adding it (and the prometheus/esgw certs) to the trusted - // bundle - and may abort the reconcile by returning an error. - renderCtx, err := extensions.RunSetup(extensions.Inputs{ + renderCtx, err := extensions.BuildContext(extensions.Inputs{ Ctx: ctx, Client: r.client, Installation: &instance.Spec, @@ -1245,7 +1239,6 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile r.status.SetDegraded(operatorv1.ResourceCreateError, "Error preparing installation extension", err, reqLogger) return reconcile.Result{}, err } - nodePrometheusTLS := renderCtx.NodePrometheusTLS kubeControllersMetricsPort, err := utils.GetKubeControllerMetricsPort(ctx, r.client) if err != nil { @@ -1284,7 +1277,13 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile } // Create a component handler to create or update the rendered components. - handler := r.newComponentHandler(log, r.client, r.scheme, instance, utils.WithRenderContext(renderCtx)) + handler := r.newComponentHandler( + log, + r.client, + r.scheme, + instance, + utils.WithRenderContext(renderCtx), + ) // Render namespaces first - this ensures that any other controllers blocked on namespace existence can proceed. namespaceCfg := &render.NamespaceConfiguration{ @@ -1377,7 +1376,7 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile keyPairOptions := []rcertificatemanagement.KeyPairOption{ rcertificatemanagement.NewKeyPairOption(typhaNodeTLS.NodeSecret, true, true), - rcertificatemanagement.NewKeyPairOption(nodePrometheusTLS, true, true), + rcertificatemanagement.NewKeyPairOption(renderCtx.NodePrometheusTLS, true, true), rcertificatemanagement.NewKeyPairOption(typhaNodeTLS.TyphaSecret, true, true), rcertificatemanagement.NewKeyPairOption(typhaNodeTLS.TyphaSecretNonClusterHost, true, true), rcertificatemanagement.NewKeyPairOption(kubeControllerTLS, true, true), @@ -1808,7 +1807,7 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile render.TyphaTLSSecretName: typhaNodeTLS.TyphaSecret, render.NodeTLSSecretName: typhaNodeTLS.NodeSecret, render.TyphaTLSSecretName + render.TyphaNonClusterHostSuffix: typhaNodeTLS.TyphaSecretNonClusterHost, - render.NodePrometheusTLSServerSecret: nodePrometheusTLS, + render.NodePrometheusTLSServerSecret: renderCtx.NodePrometheusTLS, kubecontrollers.KubeControllerPrometheusTLSSecret: kubeControllerTLS, }, r.status) diff --git a/pkg/controller/installation/windows_controller.go b/pkg/controller/installation/windows_controller.go index e3cb1389b2..ca8063ffea 100644 --- a/pkg/controller/installation/windows_controller.go +++ b/pkg/controller/installation/windows_controller.go @@ -407,10 +407,13 @@ func (r *ReconcileWindows) Reconcile(ctx context.Context, request reconcile.Requ } // Create a component handler to create or update the rendered components. - // The render context carries the installation so registered modifiers gate on - // variant; the windows component supplies its own per-component context (the - // reporter port and prometheus keypair) via ExtensionContext. - handler := utils.NewComponentHandler(logw, r.client, r.scheme, instance, utils.WithRenderContext(extensions.RenderContext{Installation: &instance.Spec})) + handler := utils.NewComponentHandler( + logw, + r.client, + r.scheme, + instance, + utils.WithRenderContext(extensions.RenderContext{Installation: &instance.Spec}), + ) if err := handler.CreateOrUpdateOrDelete(ctx, component, nil); err != nil { r.status.SetDegraded(operatorv1.ResourceUpdateError, "Error creating / updating resource", err, reqLogger) return reconcile.Result{}, err diff --git a/pkg/enterprise/apiserver.go b/pkg/enterprise/apiserver.go index d81354a915..cba83f2bfc 100644 --- a/pkg/enterprise/apiserver.go +++ b/pkg/enterprise/apiserver.go @@ -18,6 +18,7 @@ import ( "fmt" "strings" + "github.com/sirupsen/logrus" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -66,7 +67,11 @@ func registerAPIServer() { // the query server container and its volumes, audit logging on the aggregation API server // container, the Enterprise RBAC objects, and the query server port on the Service. func modifyAPIServer(ctx extensions.RenderContext, create, del []client.Object) ([]client.Object, []client.Object) { - ec, _ := ctx.Component.(render.APIServerExtensionContext) + ec, ok := ctx.Component.(render.APIServerExtensionContext) + if !ok { + logrus.Errorf("BUG: apiserver modifier got %T, want render.APIServerExtensionContext; leaving objects unchanged", ctx.Component) + return create, del + } c := &apiServer{cfg: ec.Config, calicoImage: ec.CalicoImage} if dep, ok := extensions.FindObject[*appsv1.Deployment](create, render.APIServerName); ok { @@ -146,7 +151,11 @@ func modifyAPIServer(ctx extensions.RenderContext, create, del []client.Object) // cleanupAPIServer deletes the Enterprise API server objects when running Calico, so a // cluster switched from Enterprise to Calico does not leave them behind. func cleanupAPIServer(ctx extensions.RenderContext, create, del []client.Object) ([]client.Object, []client.Object) { - ec, _ := ctx.Component.(render.APIServerExtensionContext) + ec, ok := ctx.Component.(render.APIServerExtensionContext) + if !ok { + logrus.Errorf("BUG: apiserver cleanup got %T, want render.APIServerExtensionContext; leaving objects unchanged", ctx.Component) + return create, del + } c := &apiServer{cfg: ec.Config} del = append(del, c.tigeraAPIServerClusterRole(), c.tigeraAPIServerClusterRoleBinding()) diff --git a/pkg/enterprise/guardian.go b/pkg/enterprise/guardian.go index 1d16e13b19..0187166f61 100644 --- a/pkg/enterprise/guardian.go +++ b/pkg/enterprise/guardian.go @@ -52,7 +52,11 @@ func registerGuardian() { // fail (proxy URL parsing); on failure we drop the policy entirely, matching the // core behavior of omitting it rather than installing a partial policy. func modifyGuardianPolicy(ctx extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { - gpc, _ := ctx.Component.(render.GuardianPolicyExtensionContext) + gpc, ok := ctx.Component.(render.GuardianPolicyExtensionContext) + if !ok { + logrus.Errorf("BUG: guardian policy modifier got %T, want render.GuardianPolicyExtensionContext; leaving objects unchanged", ctx.Component) + return objs, del + } policy, ok := extensions.FindObject[*v3.NetworkPolicy](objs, render.GuardianPolicyName) if !ok { @@ -212,7 +216,11 @@ func enterpriseGuardianPolicySpec(gpc render.GuardianPolicyExtensionContext) (v3 // elasticsearch/kibana service ports, the management-cluster-request cluster // role rules (which replace the OSS rules), and the CA bundle env vars. func modifyGuardian(ctx extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { - gc, _ := ctx.Component.(render.GuardianExtensionContext) + gc, ok := ctx.Component.(render.GuardianExtensionContext) + if !ok { + logrus.Errorf("BUG: guardian modifier got %T, want render.GuardianExtensionContext; leaving objects unchanged", ctx.Component) + return objs, del + } if role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, render.GuardianClusterRoleName); ok { role.Rules = guardianEnterpriseRules(gc) diff --git a/pkg/enterprise/installation_test.go b/pkg/enterprise/installation_test.go index 1ebf7bf52b..cfab0176bb 100644 --- a/pkg/enterprise/installation_test.go +++ b/pkg/enterprise/installation_test.go @@ -42,18 +42,18 @@ var _ = Describe("installation setup", func() { in.FelixConfiguration = &v3.FelixConfiguration{ Spec: v3.FelixConfigurationSpec{PrometheusReporterPort: &port}, } - _, err := extensions.RunSetup(in) + _, err := extensions.BuildContext(in) Expect(err).To(HaveOccurred()) }) It("creates the node prometheus keypair for the enterprise variant", func() { - rc, err := extensions.RunSetup(newInputs(operatorv1.CalicoEnterprise)) + rc, err := extensions.BuildContext(newInputs(operatorv1.CalicoEnterprise)) Expect(err).NotTo(HaveOccurred()) Expect(rc.NodePrometheusTLS).NotTo(BeNil()) }) It("is a no-op for the Calico variant", func() { - rc, err := extensions.RunSetup(newInputs(operatorv1.Calico)) + rc, err := extensions.BuildContext(newInputs(operatorv1.Calico)) Expect(err).NotTo(HaveOccurred()) Expect(rc.NodePrometheusTLS).To(BeNil()) }) diff --git a/pkg/enterprise/windows.go b/pkg/enterprise/windows.go index 720ea80c20..8a3e086835 100644 --- a/pkg/enterprise/windows.go +++ b/pkg/enterprise/windows.go @@ -17,6 +17,7 @@ package enterprise import ( "fmt" + "github.com/sirupsen/logrus" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -52,7 +53,11 @@ func registerWindows() { // daemonset configuration (flow/DNS log env, prometheus reporter, trusted DNS // servers, the calico log volume, and the prometheus reporter keypair mount). func modifyWindows(ctx extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { - wc, _ := ctx.Component.(render.WindowsExtensionContext) + wc, ok := ctx.Component.(render.WindowsExtensionContext) + if !ok { + logrus.Errorf("BUG: windows modifier got %T, want render.WindowsExtensionContext; leaving objects unchanged", ctx.Component) + return objs, del + } if ds, ok := extensions.FindObject[*appsv1.DaemonSet](objs, common.WindowsDaemonSetName); ok { modifyWindowsDaemonSet(ctx, wc, ds) diff --git a/pkg/extensions/doc.go b/pkg/extensions/doc.go index 73adc07fa0..a505f82fa1 100644 --- a/pkg/extensions/doc.go +++ b/pkg/extensions/doc.go @@ -25,7 +25,7 @@ // does the side-effecting work a pure render hook can't: creating certificates, // extending the trusted bundle, validating config. It returns the RenderContext // - the read-only baton passed to the render phase. Register one per variant -// with RegisterSetup; the controller runs it with RunSetup. +// with RegisterSetup; the controller runs it with BuildContext. // // Extension is the render phase: pure, per-component hooks that run after a // component builds its objects. Its Image field overrides the component's image diff --git a/pkg/extensions/setup.go b/pkg/extensions/setup.go index 136863b0ff..a6272b006c 100644 --- a/pkg/extensions/setup.go +++ b/pkg/extensions/setup.go @@ -73,9 +73,10 @@ func RegisterSetup(variant operatorv1.ProductVariant, s Setup) { setups[variant] = s } -// RunSetup runs the setup registered for the installation variant and returns -// its RenderContext, or the base render context when the variant has no setup. -func RunSetup(in Inputs) (RenderContext, error) { +// BuildContext runs the setup registered for the installation variant and +// returns its RenderContext, or the base render context when the variant has no +// setup. +func BuildContext(in Inputs) (RenderContext, error) { if in.Installation != nil { if s, ok := setups[in.Installation.Variant]; ok { return s(in) diff --git a/pkg/extensions/setup_test.go b/pkg/extensions/setup_test.go index 9967bfe209..1ba42e5567 100644 --- a/pkg/extensions/setup_test.go +++ b/pkg/extensions/setup_test.go @@ -29,7 +29,7 @@ var _ = Describe("variant setup", func() { It("returns the base render context when the variant has no setup", func() { install := &operatorv1.InstallationSpec{Variant: operatorv1.Calico} - rc, err := extensions.RunSetup(extensions.Inputs{ + rc, err := extensions.BuildContext(extensions.Inputs{ Installation: install, ClusterDomain: "cluster.local", }) @@ -41,14 +41,14 @@ var _ = Describe("variant setup", func() { It("uses the setup registered for the installation variant", func() { extensions.RegisterSetup(operatorv1.CalicoEnterprise, fakeSetup(nil)) - rc, err := extensions.RunSetup(enterpriseInputs()) + rc, err := extensions.BuildContext(enterpriseInputs()) Expect(err).NotTo(HaveOccurred()) Expect(rc.ClusterDomain).To(Equal("from-fake")) }) It("ignores a setup registered for a different variant", func() { extensions.RegisterSetup(operatorv1.CalicoEnterprise, fakeSetup(nil)) - rc, err := extensions.RunSetup(extensions.Inputs{ + rc, err := extensions.BuildContext(extensions.Inputs{ Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}, ClusterDomain: "real", }) @@ -58,7 +58,7 @@ var _ = Describe("variant setup", func() { It("surfaces the setup error", func() { extensions.RegisterSetup(operatorv1.CalicoEnterprise, fakeSetup(errors.New("boom"))) - _, err := extensions.RunSetup(enterpriseInputs()) + _, err := extensions.BuildContext(enterpriseInputs()) Expect(err).To(MatchError("boom")) }) @@ -67,7 +67,7 @@ var _ = Describe("variant setup", func() { extensions.ResetForTest() in := enterpriseInputs() in.ClusterDomain = "real" - rc, err := extensions.RunSetup(in) + rc, err := extensions.BuildContext(in) Expect(err).NotTo(HaveOccurred()) Expect(rc.ClusterDomain).To(Equal("real")) }) From 36b9881634a8a05b28dd21e4ecaee61ab7504cec Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Wed, 17 Jun 2026 13:07:07 -0700 Subject: [PATCH 26/38] Inject extensions instead of registering them globally The variant modifiers, image overrides, and setups lived in package-level registries that enterprise populated via an init-style Register() at startup. Replace those with an extensions.Set value that main builds (enterprise.New()) and hands to controllers through ControllerOptions. Each controller threads it into its component handler and the node/windows render configs, so nothing is wired by import side effect and the core operator can run with no Set. A handler that renders an extensible component without being given the Set logs a BUG, so a missing wiring is loud rather than a silent no-op. Rename imageoverride.Registry to imageoverride.Overrides so it doesn't read as the container image registry. --- cmd/main.go | 10 +-- .../apiserver/apiserver_controller.go | 1 + .../apiserver/apiserver_controller_test.go | 21 +++++ .../apiserver/apiserver_suite_test.go | 10 ++- .../clusterconnection_controller.go | 3 + .../clusterconnection_suite_test.go | 6 -- pkg/controller/clusterconnection/shim_test.go | 2 + .../installation/core_controller.go | 6 +- .../installation/core_controller_test.go | 22 +++++ .../installation_controller_suite_test.go | 9 +- .../installation/windows_controller.go | 4 + .../installation/windows_controller_test.go | 3 + pkg/controller/options/options.go | 7 ++ pkg/controller/utils/component.go | 23 +++++- .../utils/component_enterprise_test.go | 5 +- pkg/controller/utils/component_test.go | 9 +- pkg/enterprise/apiserver.go | 6 +- pkg/enterprise/enterprise_suite_test.go | 7 ++ pkg/enterprise/guardian.go | 6 +- pkg/enterprise/guardian_test.go | 13 ++- pkg/enterprise/installation.go | 4 +- pkg/enterprise/installation_test.go | 9 +- pkg/enterprise/node.go | 6 +- pkg/enterprise/node_test.go | 29 +++---- pkg/enterprise/register.go | 25 +++--- pkg/enterprise/typha.go | 4 +- pkg/enterprise/typha_test.go | 11 +-- pkg/enterprise/windows.go | 8 +- pkg/enterprise/windows_test.go | 23 ++---- pkg/extensions/extension.go | 35 ++------ pkg/extensions/extension_test.go | 27 +++--- pkg/extensions/image.go | 9 -- pkg/extensions/image_test.go | 22 ++--- pkg/extensions/set.go | 82 +++++++++++++++++++ pkg/extensions/setup.go | 20 ++--- pkg/extensions/setup_test.go | 26 +++--- pkg/imageoverride/imageoverride.go | 34 ++++---- pkg/render/apiserver_test.go | 2 +- pkg/render/enterprise_setup_test.go | 21 ++--- pkg/render/guardian_test.go | 6 +- pkg/render/node.go | 9 +- pkg/render/node_enterprise_test.go | 4 +- pkg/render/node_test.go | 1 + pkg/render/windows.go | 9 +- pkg/render/windows_test.go | 15 ++-- 45 files changed, 366 insertions(+), 248 deletions(-) create mode 100644 pkg/extensions/set.go diff --git a/cmd/main.go b/cmd/main.go index 346f62a8a1..eb63d9b666 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -522,6 +522,11 @@ If a value other than 'all' is specified, the first CRD with a prefix of the spe ElasticExternal: discovery.UseExternalElastic(bootConfig), UseV3CRDs: v3CRDs, APIDiscovery: apiDiscovery, + // Hand the operator the in-repo Calico Enterprise extensions (modifiers, + // image overrides, and the installation setup). After the monorepo split + // the core operator's main passes none and calico-private's main passes + // its own. + Extensions: enterprise.New(), } // Before we start any controllers, make sure our options are valid. @@ -530,11 +535,6 @@ If a value other than 'all' is specified, the first CRD with a prefix of the spe os.Exit(1) } - // Wire the in-repo Calico Enterprise extensions (the render context factory, - // modifiers, and image overrides) into the operator registries. After the - // monorepo split this call moves to calico-private's main. - enterprise.Register() - // Register a field-selector index on Pod spec.nodeName. The podiprecovery // controller uses this to list operator-managed pods on a specific node // in a single server-side query. Indexes must be registered before the diff --git a/pkg/controller/apiserver/apiserver_controller.go b/pkg/controller/apiserver/apiserver_controller.go index 402e784920..3b8c5e4197 100644 --- a/pkg/controller/apiserver/apiserver_controller.go +++ b/pkg/controller/apiserver/apiserver_controller.go @@ -481,6 +481,7 @@ func (r *ReconcileAPIServer) Reconcile(ctx context.Context, request reconcile.Re r.scheme, instance, utils.WithRenderContext(extensions.RenderContext{Installation: installationSpec}), + utils.WithExtensions(r.opts.Extensions), ) // Render the desired objects from the CRD and create or update them. diff --git a/pkg/controller/apiserver/apiserver_controller_test.go b/pkg/controller/apiserver/apiserver_controller_test.go index b70bd58906..11a4f896de 100644 --- a/pkg/controller/apiserver/apiserver_controller_test.go +++ b/pkg/controller/apiserver/apiserver_controller_test.go @@ -170,6 +170,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, }, @@ -229,6 +230,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, }, @@ -282,6 +284,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, ClusterDomain: dns.DefaultClusterDomain, @@ -307,6 +310,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, }, @@ -329,6 +333,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, }, @@ -353,6 +358,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, }, @@ -375,6 +381,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: notReady, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, }, @@ -400,6 +407,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, }, @@ -427,6 +435,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, }, @@ -452,6 +461,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: notReady, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, }, @@ -478,6 +488,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: false, DetectedProvider: operatorv1.ProviderNone, }, @@ -520,6 +531,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, }, @@ -552,6 +564,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, }, @@ -604,6 +617,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, }, @@ -673,6 +687,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, }, @@ -777,6 +792,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, }, @@ -806,6 +822,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, }, @@ -836,6 +853,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, MultiTenant: true, @@ -883,6 +901,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, UseV3CRDs: true, @@ -927,6 +946,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: false, DetectedProvider: operatorv1.ProviderNone, UseV3CRDs: true, @@ -955,6 +975,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, UseV3CRDs: false, diff --git a/pkg/controller/apiserver/apiserver_suite_test.go b/pkg/controller/apiserver/apiserver_suite_test.go index 17f3e98c8d..2e1863335b 100644 --- a/pkg/controller/apiserver/apiserver_suite_test.go +++ b/pkg/controller/apiserver/apiserver_suite_test.go @@ -26,12 +26,16 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log/zap" "github.com/tigera/operator/pkg/enterprise" + "github.com/tigera/operator/pkg/extensions" ) +// testExtensions is the enterprise extension Set the API server controller tests +// reconcile with, so the componentHandler applies the API server modifier (query +// server, audit logging, Enterprise RBAC). Reconcilers built in these tests put +// it on their options, mirroring how main wires it in production. +var testExtensions *extensions.Set = enterprise.New() + func TestStatus(t *testing.T) { - // Wire the enterprise extensions so the componentHandler applies the API server - // modifier (query server, audit logging, Enterprise RBAC) during reconciliation. - enterprise.Register() logf.SetLogger(zap.New(zap.WriteTo(ginkgo.GinkgoWriter), zap.UseDevMode(true), zap.Level(uzap.NewAtomicLevelAt(uzap.DebugLevel)))) gomega.RegisterFailHandler(ginkgo.Fail) suiteConfig, reporterConfig := ginkgo.GinkgoConfiguration() diff --git a/pkg/controller/clusterconnection/clusterconnection_controller.go b/pkg/controller/clusterconnection/clusterconnection_controller.go index d796e01638..763743789f 100644 --- a/pkg/controller/clusterconnection/clusterconnection_controller.go +++ b/pkg/controller/clusterconnection/clusterconnection_controller.go @@ -177,6 +177,7 @@ func newReconciler( clusterDomain: opts.ClusterDomain, tierWatchReady: tierWatchReady, clusterInfoWatchReady: clusterInfoWatchReady, + opts: opts, } c.status.Run(opts.ShutdownContext) return c @@ -196,6 +197,7 @@ type ReconcileConnection struct { clusterInfoWatchReady *utils.ReadyFlag resolvedPodProxies []*httpproxy.Config lastAvailabilityTransition metav1.Time + opts options.ControllerOptions } // Reconcile reads that state of the cluster for a ManagementClusterConnection object and makes changes based on the @@ -450,6 +452,7 @@ func (r *ReconcileConnection) Reconcile(ctx context.Context, request reconcile.R r.scheme, managementClusterConnection, utils.WithRenderContext(extensions.RenderContext{Installation: installationSpec}), + utils.WithExtensions(r.opts.Extensions), ) guardianCfg := &render.GuardianConfiguration{ URL: managementClusterConnection.Spec.ManagementClusterAddr, diff --git a/pkg/controller/clusterconnection/clusterconnection_suite_test.go b/pkg/controller/clusterconnection/clusterconnection_suite_test.go index 09a8254690..2e6b6feb43 100644 --- a/pkg/controller/clusterconnection/clusterconnection_suite_test.go +++ b/pkg/controller/clusterconnection/clusterconnection_suite_test.go @@ -22,18 +22,12 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log/zap" logf "sigs.k8s.io/controller-runtime/pkg/log" - - "github.com/tigera/operator/pkg/enterprise" ) func TestStatus(t *testing.T) { logf.SetLogger(zap.New(zap.WriteTo(ginkgo.GinkgoWriter))) gomega.RegisterFailHandler(ginkgo.Fail) - // Wire the enterprise extensions so the guardian modifier runs the way it - // does in the operator binary, which is what these controller tests exercise. - enterprise.Register() - suiteConfig, reporterConfig := ginkgo.GinkgoConfiguration() reporterConfig.JUnitReport = "../../../report/ut/clusterconnection_controller_suite.xml" ginkgo.RunSpecs(t, "pkg/controller/Management Cluster Connection Suite", suiteConfig, reporterConfig) diff --git a/pkg/controller/clusterconnection/shim_test.go b/pkg/controller/clusterconnection/shim_test.go index 7446d229ac..bd5993478d 100644 --- a/pkg/controller/clusterconnection/shim_test.go +++ b/pkg/controller/clusterconnection/shim_test.go @@ -25,6 +25,7 @@ import ( operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/controller/options" "github.com/tigera/operator/pkg/controller/status" + "github.com/tigera/operator/pkg/enterprise" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -40,6 +41,7 @@ func NewReconcilerWithShims( ) reconcile.Reconciler { opts := options.ControllerOptions{ ShutdownContext: context.Background(), + Extensions: enterprise.New(), } return newReconciler(cli, schema, status, provider, tierWatchReady, clusterInfoWatchReady, opts) diff --git a/pkg/controller/installation/core_controller.go b/pkg/controller/installation/core_controller.go index d760aed637..1f241da0a8 100644 --- a/pkg/controller/installation/core_controller.go +++ b/pkg/controller/installation/core_controller.go @@ -355,6 +355,7 @@ func newReconciler(mgr manager.Manager, opts options.ControllerOptions) (*Reconc v3CRDs: opts.UseV3CRDs, kubernetesVersion: opts.KubernetesVersion, apiDiscovery: opts.APIDiscovery, + opts: opts, } r.status.Run(opts.ShutdownContext) r.typhaAutoscaler.start(opts.ShutdownContext) @@ -414,6 +415,7 @@ type ReconcileInstallation struct { v3CRDs bool kubernetesVersion *common.VersionInfo apiDiscovery *discovery.APIDiscovery + opts options.ControllerOptions // newComponentHandler returns a new component handler. Useful stub for unit testing. newComponentHandler func(log logr.Logger, client client.Client, scheme *runtime.Scheme, cr metav1.Object, opts ...utils.ComponentHandlerOption) utils.ComponentHandler @@ -1226,7 +1228,7 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile calicoVersion = components.EnterpriseRelease } - renderCtx, err := extensions.BuildContext(extensions.Inputs{ + renderCtx, err := r.opts.Extensions.BuildContext(extensions.Inputs{ Ctx: ctx, Client: r.client, Installation: &instance.Spec, @@ -1283,6 +1285,7 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile r.scheme, instance, utils.WithRenderContext(renderCtx), + utils.WithExtensions(r.opts.Extensions), ) // Render namespaces first - this ensures that any other controllers blocked on namespace existence can proceed. @@ -1576,6 +1579,7 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile FelixHealthPort: *felixConfiguration.Spec.HealthPort, NodeCgroupV2Path: felixConfiguration.Spec.CgroupV2Path, V3CRDs: r.v3CRDs, + ImageOverrides: r.opts.Extensions.Images(), } if bgpConfiguration.Spec.BindMode != nil { diff --git a/pkg/controller/installation/core_controller_test.go b/pkg/controller/installation/core_controller_test.go index 39e171090a..d2e73fe0ad 100644 --- a/pkg/controller/installation/core_controller_test.go +++ b/pkg/controller/installation/core_controller_test.go @@ -56,6 +56,7 @@ import ( "github.com/tigera/operator/pkg/common/discovery" "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/controller/options" "github.com/tigera/operator/pkg/controller/status" "github.com/tigera/operator/pkg/controller/utils" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" @@ -190,6 +191,7 @@ var _ = Describe("Testing core-controller installation", func() { // As the parameters in the client changes, we expect the outcomes of the reconcile loops to change. r = ReconcileInstallation{ + opts: options.ControllerOptions{Extensions: testExtensions}, config: nil, // there is no fake for config client: c, scheme: scheme, @@ -819,6 +821,7 @@ var _ = Describe("Testing core-controller installation", func() { // As the parameters in the client changes, we expect the outcomes of the reconcile loops to change. r = ReconcileInstallation{ + opts: options.ControllerOptions{Extensions: testExtensions}, config: nil, // there is no fake for config client: c, scheme: scheme, @@ -1041,6 +1044,7 @@ var _ = Describe("Testing core-controller installation", func() { // As the parameters in the client changes, we expect the outcomes of the reconcile loops to change. r = ReconcileInstallation{ + opts: options.ControllerOptions{Extensions: testExtensions}, config: nil, // there is no fake for config client: c, scheme: scheme, @@ -2329,6 +2333,7 @@ var _ = Describe("Testing core-controller installation", func() { // As the parameters in the client changes, we expect the outcomes of the reconcile loops to change. r = ReconcileInstallation{ + opts: options.ControllerOptions{Extensions: testExtensions}, config: nil, // there is no fake for config client: c, scheme: scheme, @@ -2466,6 +2471,7 @@ var _ = Describe("Testing core-controller installation", func() { componentHandler = newFakeComponentHandler() r = ReconcileInstallation{ + opts: options.ControllerOptions{Extensions: testExtensions}, config: nil, // there is no fake for config client: c, scheme: scheme, @@ -2629,6 +2635,7 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { It("should create v1 MAPs when v1 is served", func() { r = ReconcileInstallation{ + opts: options.ControllerOptions{Extensions: testExtensions}, client: clientFor(), scheme: scheme, status: mockStatus, @@ -2660,6 +2667,7 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { It("should create v1beta1 MAPs when only v1beta1 is served", func() { r = ReconcileInstallation{ + opts: options.ControllerOptions{Extensions: testExtensions}, client: clientFor(), scheme: scheme, status: mockStatus, @@ -2689,6 +2697,7 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { It("should create v1alpha1 MAPs when only v1alpha1 is served", func() { r = ReconcileInstallation{ + opts: options.ControllerOptions{Extensions: testExtensions}, client: clientFor(), scheme: scheme, status: mockStatus, @@ -2718,6 +2727,7 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { It("should not create MAPs when no served version exists and should set degraded", func() { r = ReconcileInstallation{ + opts: options.ControllerOptions{Extensions: testExtensions}, client: clientFor(), scheme: scheme, status: mockStatus, @@ -2736,6 +2746,7 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { It("should not create MAPs when v3CRDs=false", func() { r = ReconcileInstallation{ + opts: options.ControllerOptions{Extensions: testExtensions}, client: clientFor(), scheme: scheme, status: mockStatus, @@ -2753,6 +2764,7 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { It("should not create MAPs when manageCRDs=false", func() { r = ReconcileInstallation{ + opts: options.ControllerOptions{Extensions: testExtensions}, client: clientFor(), scheme: scheme, status: mockStatus, @@ -2783,6 +2795,7 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { } r = ReconcileInstallation{ + opts: options.ControllerOptions{Extensions: testExtensions}, client: clientFor(staleMAP, staleMAPB), scheme: scheme, status: mockStatus, @@ -2825,6 +2838,7 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { } r = ReconcileInstallation{ + opts: options.ControllerOptions{Extensions: testExtensions}, client: clientFor(initial...), scheme: scheme, status: mockStatus, @@ -2843,6 +2857,7 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { It("should work with Enterprise variant", func() { r = ReconcileInstallation{ + opts: options.ControllerOptions{Extensions: testExtensions}, client: clientFor(), scheme: scheme, status: mockStatus, @@ -2914,6 +2929,7 @@ var _ = Describe("updateValidatingAdmissionPolicies", func() { It("should create v1 VAPs when v1 is served", func() { r = ReconcileInstallation{ + opts: options.ControllerOptions{Extensions: testExtensions}, client: clientFor(), scheme: scheme, status: mockStatus, @@ -2945,6 +2961,7 @@ var _ = Describe("updateValidatingAdmissionPolicies", func() { It("should create v1beta1 VAPs when only v1beta1 is served", func() { r = ReconcileInstallation{ + opts: options.ControllerOptions{Extensions: testExtensions}, client: clientFor(), scheme: scheme, status: mockStatus, @@ -2974,6 +2991,7 @@ var _ = Describe("updateValidatingAdmissionPolicies", func() { It("should create v1alpha1 VAPs when only v1alpha1 is served", func() { r = ReconcileInstallation{ + opts: options.ControllerOptions{Extensions: testExtensions}, client: clientFor(), scheme: scheme, status: mockStatus, @@ -2991,6 +3009,7 @@ var _ = Describe("updateValidatingAdmissionPolicies", func() { It("should skip without degrading when no served version exists", func() { r = ReconcileInstallation{ + opts: options.ControllerOptions{Extensions: testExtensions}, client: clientFor(), scheme: scheme, status: mockStatus, @@ -3009,6 +3028,7 @@ var _ = Describe("updateValidatingAdmissionPolicies", func() { It("should not create VAPs when v3CRDs=false", func() { r = ReconcileInstallation{ + opts: options.ControllerOptions{Extensions: testExtensions}, client: clientFor(), scheme: scheme, status: mockStatus, @@ -3039,6 +3059,7 @@ var _ = Describe("updateValidatingAdmissionPolicies", func() { } r = ReconcileInstallation{ + opts: options.ControllerOptions{Extensions: testExtensions}, client: clientFor(staleVAP, staleVAPB), scheme: scheme, status: mockStatus, @@ -3063,6 +3084,7 @@ var _ = Describe("updateValidatingAdmissionPolicies", func() { It("should work with Enterprise variant", func() { r = ReconcileInstallation{ + opts: options.ControllerOptions{Extensions: testExtensions}, client: clientFor(), scheme: scheme, status: mockStatus, diff --git a/pkg/controller/installation/installation_controller_suite_test.go b/pkg/controller/installation/installation_controller_suite_test.go index 778ee91f8f..f589361c5a 100644 --- a/pkg/controller/installation/installation_controller_suite_test.go +++ b/pkg/controller/installation/installation_controller_suite_test.go @@ -27,11 +27,14 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log/zap" "github.com/tigera/operator/pkg/enterprise" + "github.com/tigera/operator/pkg/extensions" ) -var _ = ginkgo.BeforeSuite(func() { - enterprise.Register() -}) +// testExtensions is the enterprise extension Set the installation controller +// tests reconcile with, mirroring how main wires it in production. Reconcilers +// built in these tests put it on their options so the node image overrides and +// modifiers apply. +var testExtensions *extensions.Set = enterprise.New() func TestInstallation(t *testing.T) { // Disable WatchListClient for tests. In client-go v0.35+, this feature defaults to true and diff --git a/pkg/controller/installation/windows_controller.go b/pkg/controller/installation/windows_controller.go index ca8063ffea..12e31b2a1f 100644 --- a/pkg/controller/installation/windows_controller.go +++ b/pkg/controller/installation/windows_controller.go @@ -182,6 +182,7 @@ type ReconcileWindows struct { enterpriseCRDsExist bool clusterDomain string ipamConfigWatchReady *utils.ReadyFlag + opts options.ControllerOptions } // newWindowsReconciler returns a new reconcile.Reconciler @@ -198,6 +199,7 @@ func newWindowsReconciler(mgr manager.Manager, opts options.ControllerOptions) ( enterpriseCRDsExist: opts.EnterpriseCRDExists, clusterDomain: opts.ClusterDomain, ipamConfigWatchReady: &utils.ReadyFlag{}, + opts: opts, } r.status.Run(opts.ShutdownContext) return r, nil @@ -387,6 +389,7 @@ func (r *ReconcileWindows) Reconcile(ctx context.Context, request reconcile.Requ PrometheusServerTLS: nodePrometheusTLS, NodeReporterMetricsPort: nodeReporterMetricsPort, VXLANVNI: *felixConfiguration.Spec.VXLANVNI, + ImageOverrides: r.opts.Extensions.Images(), } component = render.Windows(&windowsCfg) @@ -413,6 +416,7 @@ func (r *ReconcileWindows) Reconcile(ctx context.Context, request reconcile.Requ r.scheme, instance, utils.WithRenderContext(extensions.RenderContext{Installation: &instance.Spec}), + utils.WithExtensions(r.opts.Extensions), ) if err := handler.CreateOrUpdateOrDelete(ctx, component, nil); err != nil { r.status.SetDegraded(operatorv1.ResourceUpdateError, "Error creating / updating resource", err, reqLogger) diff --git a/pkg/controller/installation/windows_controller_test.go b/pkg/controller/installation/windows_controller_test.go index ae5866bfa5..98121210e7 100644 --- a/pkg/controller/installation/windows_controller_test.go +++ b/pkg/controller/installation/windows_controller_test.go @@ -29,6 +29,7 @@ import ( "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/controller/options" "github.com/tigera/operator/pkg/controller/status" "github.com/tigera/operator/pkg/controller/utils" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" @@ -119,6 +120,7 @@ var _ = Describe("windows-controller installation tests", func() { // As the parameters in the client changes, we expect the outcomes of the reconcile loops to change. r = ReconcileWindows{ + opts: options.ControllerOptions{Extensions: testExtensions}, config: nil, // there is no fake for config client: c, scheme: scheme, @@ -609,6 +611,7 @@ var _ = Describe("windows-controller installation tests", func() { // As the parameters in the client changes, we expect the outcomes of the reconcile loops to change. r = ReconcileWindows{ + opts: options.ControllerOptions{Extensions: testExtensions}, config: nil, // there is no fake for config client: c, scheme: scheme, diff --git a/pkg/controller/options/options.go b/pkg/controller/options/options.go index 14acd47230..404c399378 100644 --- a/pkg/controller/options/options.go +++ b/pkg/controller/options/options.go @@ -20,6 +20,7 @@ import ( v1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/common/discovery" + "github.com/tigera/operator/pkg/extensions" "k8s.io/client-go/kubernetes" ) @@ -55,4 +56,10 @@ type ControllerOptions struct { // the operator cares about. Populated once at startup so controllers can branch on API // availability without issuing further discovery requests at reconcile time. APIDiscovery *discovery.APIDiscovery + + // Extensions are the variant extensions (modifiers, image overrides, setups) + // the operator runs with. The core operator leaves this nil; an extension + // build (Calico Enterprise) sets it once at startup and controllers thread it + // into their render handlers and component configs. + Extensions *extensions.Set } diff --git a/pkg/controller/utils/component.go b/pkg/controller/utils/component.go index 885ee21da3..76db4ebf44 100644 --- a/pkg/controller/utils/component.go +++ b/pkg/controller/utils/component.go @@ -84,6 +84,13 @@ func WithRenderContext(ctx extensions.RenderContext) ComponentHandlerOption { return func(c *componentHandler) { c.renderCtx = ctx } } +// WithExtensions supplies the operator's extension Set, whose modifiers the +// handler applies to extensible components. A handler that renders an +// extensible component must be given the Set; one that doesn't can omit it. +func WithExtensions(e *extensions.Set) ComponentHandlerOption { + return func(c *componentHandler) { c.extensions = e } +} + // cr is allowed to be nil in the case we don't want to put ownership on a resource, // this is useful for CRD management so that they are not removed automatically. func NewComponentHandler(log logr.Logger, cli client.Client, scheme *runtime.Scheme, cr metav1.Object, opts ...ComponentHandlerOption) ComponentHandler { @@ -108,6 +115,7 @@ type componentHandler struct { createOnly bool apiGroupEnvs []v1.EnvVar renderCtx extensions.RenderContext + extensions *extensions.Set } func (c *componentHandler) SetCreateOnly() { @@ -469,11 +477,18 @@ func (c *componentHandler) CreateOrUpdateOrDelete(ctx context.Context, component objsToCreate, objsToDelete := component.Objects() if ext, ok := component.(render.Extensible); ok { - rc := c.renderCtx - if p, ok := component.(render.ExtensionContextProvider); ok { - rc.Component = p.ExtensionContext() + if c.extensions == nil { + // The component can be extended but this handler was built without an + // extension Set, so any registered modifier silently won't run. That is + // a wiring bug in the controller, not a normal state. + c.log.Info("BUG: extensible component rendered by a handler with no extension Set; modifiers will not be applied", "component", ext.ModifierKey()) + } else { + rc := c.renderCtx + if p, ok := component.(render.ExtensionContextProvider); ok { + rc.Component = p.ExtensionContext() + } + objsToCreate, objsToDelete = c.extensions.ApplyModifiers(ext.ModifierKey(), rc, objsToCreate, objsToDelete) } - objsToCreate, objsToDelete = extensions.ApplyModifiers(ext.ModifierKey(), rc, objsToCreate, objsToDelete) } // Load the InstallationSpec once and reuse it for every object: createOrUpdateObject needs it diff --git a/pkg/controller/utils/component_enterprise_test.go b/pkg/controller/utils/component_enterprise_test.go index a59e1394b6..12d63fa44c 100644 --- a/pkg/controller/utils/component_enterprise_test.go +++ b/pkg/controller/utils/component_enterprise_test.go @@ -43,9 +43,6 @@ import ( // registered modifier must match the real render output by name. If render ever // renames the typha ClusterRole, the modifier silently no-ops and this fails. var _ = Describe("componentHandler enterprise modifier integration", func() { - BeforeEach(func() { enterprise.Register() }) - AfterEach(func() { extensions.ResetForTest() }) - It("applies the enterprise typha modifier to real render output", func() { scheme := runtime.NewScheme() Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) @@ -77,7 +74,7 @@ var _ = Describe("componentHandler enterprise modifier integration", func() { }) renderCtx := extensions.RenderContext{Installation: instance} - handler := utils.NewComponentHandler(logf.Log, cli, scheme, nil, utils.WithRenderContext(renderCtx)) + handler := utils.NewComponentHandler(logf.Log, cli, scheme, nil, utils.WithRenderContext(renderCtx), utils.WithExtensions(enterprise.New())) Expect(handler.CreateOrUpdateOrDelete(context.Background(), comp, nil)).NotTo(HaveOccurred()) role := &rbacv1.ClusterRole{} diff --git a/pkg/controller/utils/component_test.go b/pkg/controller/utils/component_test.go index 9790fd5939..d8c227d47b 100644 --- a/pkg/controller/utils/component_test.go +++ b/pkg/controller/utils/component_test.go @@ -2578,12 +2578,9 @@ func (mc *mockClient) SubResource(subResource string) client.SubResourceClient { } var _ = Describe("componentHandler modifier application", func() { - AfterEach(func() { - extensions.ResetForTest() - }) - It("applies registered modifiers to a named component before create", func() { - extensions.Register(operatorv1.CalicoEnterprise, "fake", extensions.Extension{ + ext := extensions.NewSet() + ext.Register(operatorv1.CalicoEnterprise, "fake", extensions.Extension{ Modify: func(ctx extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { cm := objs[0].(*corev1.ConfigMap) cm.Data = map[string]string{"patched": "yes"} @@ -2597,7 +2594,7 @@ var _ = Describe("componentHandler modifier application", func() { c := ctrlrfake.DefaultFakeClientBuilder(s).Build() renderCtx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}} - handler := NewComponentHandler(logf.Log, c, s, nil, WithRenderContext(renderCtx)) + handler := NewComponentHandler(logf.Log, c, s, nil, WithRenderContext(renderCtx), WithExtensions(ext)) comp := &namedFakeComponent{name: "fake", obj: &corev1.ConfigMap{ TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: "cm", Namespace: "default"}, diff --git a/pkg/enterprise/apiserver.go b/pkg/enterprise/apiserver.go index cba83f2bfc..1b7119f239 100644 --- a/pkg/enterprise/apiserver.go +++ b/pkg/enterprise/apiserver.go @@ -52,13 +52,13 @@ type apiServer struct { calicoImage string } -func registerAPIServer() { - extensions.Register(operatorv1.CalicoEnterprise, render.ComponentNameAPIServer, extensions.Extension{ +func registerAPIServer(s *extensions.Set) { + s.Register(operatorv1.CalicoEnterprise, render.ComponentNameAPIServer, extensions.Extension{ Modify: modifyAPIServer, }) // When running Calico, clean up any Enterprise objects left behind by a prior // Enterprise installation. - extensions.Register(operatorv1.Calico, render.ComponentNameAPIServer, extensions.Extension{ + s.Register(operatorv1.Calico, render.ComponentNameAPIServer, extensions.Extension{ Modify: cleanupAPIServer, }) } diff --git a/pkg/enterprise/enterprise_suite_test.go b/pkg/enterprise/enterprise_suite_test.go index e7cfab281b..368107b5a0 100644 --- a/pkg/enterprise/enterprise_suite_test.go +++ b/pkg/enterprise/enterprise_suite_test.go @@ -19,8 +19,15 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + + "github.com/tigera/operator/pkg/enterprise" + "github.com/tigera/operator/pkg/extensions" ) +// ext is the enterprise extension Set under test, shared across the suite. It is +// immutable once built and the specs only read it, so a single instance is safe. +var ext *extensions.Set = enterprise.New() + func TestEnterprise(t *testing.T) { RegisterFailHandler(Fail) RunSpecs(t, "pkg/enterprise Suite") diff --git a/pkg/enterprise/guardian.go b/pkg/enterprise/guardian.go index 0187166f61..cff8185428 100644 --- a/pkg/enterprise/guardian.go +++ b/pkg/enterprise/guardian.go @@ -38,11 +38,11 @@ import ( operatorurl "github.com/tigera/operator/pkg/url" ) -func registerGuardian() { - extensions.Register(operatorv1.CalicoEnterprise, render.GuardianName, extensions.Extension{ +func registerGuardian(s *extensions.Set) { + s.Register(operatorv1.CalicoEnterprise, render.GuardianName, extensions.Extension{ Modify: modifyGuardian, }) - extensions.Register(operatorv1.CalicoEnterprise, render.ComponentNameGuardianPolicy, extensions.Extension{ + s.Register(operatorv1.CalicoEnterprise, render.ComponentNameGuardianPolicy, extensions.Extension{ Modify: modifyGuardianPolicy, }) } diff --git a/pkg/enterprise/guardian_test.go b/pkg/enterprise/guardian_test.go index 9b43423199..5223f72a47 100644 --- a/pkg/enterprise/guardian_test.go +++ b/pkg/enterprise/guardian_test.go @@ -26,14 +26,11 @@ import ( v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" operatorv1 "github.com/tigera/operator/api/v1" - "github.com/tigera/operator/pkg/enterprise" "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" ) var _ = Describe("guardian enterprise modifier", func() { - BeforeEach(func() { enterprise.Register() }) - AfterEach(func() { extensions.ResetForTest() }) // newObjs returns the subset of rendered guardian objects the modifier touches. newObjs := func() []client.Object { @@ -60,7 +57,7 @@ var _ = Describe("guardian enterprise modifier", func() { } It("appends the secrets RBAC and UI settings", func() { - out, _ := extensions.ApplyModifiers(render.GuardianName, ctxWith(render.GuardianExtensionContext{}), newObjs(), nil) + out, _ := ext.ApplyModifiers(render.GuardianName, ctxWith(render.GuardianExtensionContext{}), newObjs(), nil) _, ok := extensions.FindObject[*rbacv1.Role](out, render.GuardianSecretsRole) Expect(ok).To(BeTrue()) _, ok = extensions.FindObject[*rbacv1.RoleBinding](out, render.GuardianSecretsRoleBindingName) @@ -70,7 +67,7 @@ var _ = Describe("guardian enterprise modifier", func() { }) It("adds the elasticsearch and kibana service ports", func() { - out, _ := extensions.ApplyModifiers(render.GuardianName, ctxWith(render.GuardianExtensionContext{}), newObjs(), nil) + out, _ := ext.ApplyModifiers(render.GuardianName, ctxWith(render.GuardianExtensionContext{}), newObjs(), nil) svc, _ := extensions.FindObject[*corev1.Service](out, render.GuardianServiceName) names := []string{} for _, p := range svc.Spec.Ports { @@ -83,7 +80,7 @@ var _ = Describe("guardian enterprise modifier", func() { gc := render.GuardianExtensionContext{ Impersonation: &operatorv1.Impersonation{Users: []string{"foo"}, Groups: []string{"bar"}}, } - out, _ := extensions.ApplyModifiers(render.GuardianName, ctxWith(gc), newObjs(), nil) + out, _ := ext.ApplyModifiers(render.GuardianName, ctxWith(gc), newObjs(), nil) role, _ := extensions.FindObject[*rbacv1.ClusterRole](out, render.GuardianClusterRoleName) // The single OSS placeholder rule is gone, replaced by the enterprise set. @@ -94,14 +91,14 @@ var _ = Describe("guardian enterprise modifier", func() { It("adds the CA bundle env to the guardian container", func() { gc := render.GuardianExtensionContext{TrustedBundleMountPath: "/ca/bundle"} - out, _ := extensions.ApplyModifiers(render.GuardianName, ctxWith(gc), newObjs(), nil) + out, _ := ext.ApplyModifiers(render.GuardianName, ctxWith(gc), newObjs(), nil) dep, _ := extensions.FindObject[*appsv1.Deployment](out, render.GuardianDeploymentName) Expect(dep.Spec.Template.Spec.Containers[0].Env).To(ContainElement(corev1.EnvVar{Name: "GUARDIAN_PROMETHEUS_CA_BUNDLE_PATH", Value: "/ca/bundle"})) }) It("does nothing for the Calico variant", func() { ctx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}} - out, _ := extensions.ApplyModifiers(render.GuardianName, ctx, newObjs(), nil) + out, _ := ext.ApplyModifiers(render.GuardianName, ctx, newObjs(), nil) Expect(out).To(HaveLen(len(newObjs()))) role, _ := extensions.FindObject[*rbacv1.ClusterRole](out, render.GuardianClusterRoleName) Expect(role.Rules).To(Equal([]rbacv1.PolicyRule{{Verbs: []string{"get"}}})) diff --git a/pkg/enterprise/installation.go b/pkg/enterprise/installation.go index ffb0dd7c98..30f1c96341 100644 --- a/pkg/enterprise/installation.go +++ b/pkg/enterprise/installation.go @@ -27,8 +27,8 @@ import ( "github.com/tigera/operator/pkg/render/monitor" ) -func registerInstallation() { - extensions.RegisterSetup(operatorv1.CalicoEnterprise, setup) +func registerInstallation(s *extensions.Set) { + s.RegisterSetup(operatorv1.CalicoEnterprise, setup) } // setup is the Calico Enterprise setup phase. It builds the base render context diff --git a/pkg/enterprise/installation_test.go b/pkg/enterprise/installation_test.go index cfab0176bb..bc704a2429 100644 --- a/pkg/enterprise/installation_test.go +++ b/pkg/enterprise/installation_test.go @@ -28,13 +28,10 @@ import ( "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/controller/certificatemanager" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" - "github.com/tigera/operator/pkg/enterprise" "github.com/tigera/operator/pkg/extensions" ) var _ = Describe("installation setup", func() { - BeforeEach(func() { enterprise.Register() }) - AfterEach(func() { extensions.ResetForTest() }) It("rejects a zero prometheus reporter port", func() { port := 0 @@ -42,18 +39,18 @@ var _ = Describe("installation setup", func() { in.FelixConfiguration = &v3.FelixConfiguration{ Spec: v3.FelixConfigurationSpec{PrometheusReporterPort: &port}, } - _, err := extensions.BuildContext(in) + _, err := ext.BuildContext(in) Expect(err).To(HaveOccurred()) }) It("creates the node prometheus keypair for the enterprise variant", func() { - rc, err := extensions.BuildContext(newInputs(operatorv1.CalicoEnterprise)) + rc, err := ext.BuildContext(newInputs(operatorv1.CalicoEnterprise)) Expect(err).NotTo(HaveOccurred()) Expect(rc.NodePrometheusTLS).NotTo(BeNil()) }) It("is a no-op for the Calico variant", func() { - rc, err := extensions.BuildContext(newInputs(operatorv1.Calico)) + rc, err := ext.BuildContext(newInputs(operatorv1.Calico)) Expect(err).NotTo(HaveOccurred()) Expect(rc.NodePrometheusTLS).To(BeNil()) }) diff --git a/pkg/enterprise/node.go b/pkg/enterprise/node.go index 39b6ff69c2..752155bd01 100644 --- a/pkg/enterprise/node.go +++ b/pkg/enterprise/node.go @@ -46,8 +46,8 @@ const ( installCNIContainerName = "install-cni" ) -func registerNode() { - extensions.Register(operatorv1.CalicoEnterprise, render.ComponentNameNode, extensions.Extension{ +func registerNode(s *extensions.Set) { + s.Register(operatorv1.CalicoEnterprise, render.ComponentNameNode, extensions.Extension{ Image: func(in *operatorv1.InstallationSpec) components.Component { return components.ComponentTigeraNode }, @@ -56,7 +56,7 @@ func registerNode() { // The node component renders the cni-plugins init container; its image // resolves through its own override key. - extensions.Register(operatorv1.CalicoEnterprise, render.ComponentNameCNIPlugins, extensions.Extension{ + s.Register(operatorv1.CalicoEnterprise, render.ComponentNameCNIPlugins, extensions.Extension{ Image: func(in *operatorv1.InstallationSpec) components.Component { return components.ComponentTigeraCNIPlugins }, diff --git a/pkg/enterprise/node_test.go b/pkg/enterprise/node_test.go index 22a5408754..23ca58c98e 100644 --- a/pkg/enterprise/node_test.go +++ b/pkg/enterprise/node_test.go @@ -28,29 +28,24 @@ import ( operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/components" - "github.com/tigera/operator/pkg/enterprise" "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" ) var _ = Describe("node enterprise image override", func() { - BeforeEach(func() { enterprise.Register() }) - AfterEach(func() { extensions.ResetForTest() }) It("selects the enterprise node image for the enterprise variant", func() { ent := &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise} - Expect(extensions.ResolveImage("node", components.ComponentCalicoNode, ent)).To(Equal(components.ComponentTigeraNode)) + Expect(ext.ResolveImage("node", components.ComponentCalicoNode, ent)).To(Equal(components.ComponentTigeraNode)) }) It("leaves the default in place for the Calico variant", func() { calico := &operatorv1.InstallationSpec{Variant: operatorv1.Calico} - Expect(extensions.ResolveImage("node", components.ComponentCalicoNode, calico)).To(Equal(components.ComponentCalicoNode)) + Expect(ext.ResolveImage("node", components.ComponentCalicoNode, calico)).To(Equal(components.ComponentCalicoNode)) }) }) var _ = Describe("node enterprise modifier", func() { - BeforeEach(func() { enterprise.Register() }) - AfterEach(func() { extensions.ResetForTest() }) // newObjs returns the subset of rendered node objects the modifier touches. newObjs := func() []client.Object { @@ -86,7 +81,7 @@ var _ = Describe("node enterprise modifier", func() { } It("adds the enterprise cluster role rules", func() { - out, _ := extensions.ApplyModifiers(render.ComponentNameNode, entCtx(), newObjs(), nil) + out, _ := ext.ApplyModifiers(render.ComponentNameNode, entCtx(), newObjs(), nil) nodeRole, ok := extensions.FindObject[*rbacv1.ClusterRole](out, render.CalicoNodeObjectName) Expect(ok).To(BeTrue()) @@ -98,7 +93,7 @@ var _ = Describe("node enterprise modifier", func() { }) It("adds the enterprise felix env to the node container", func() { - out, _ := extensions.ApplyModifiers(render.ComponentNameNode, entCtx(), newObjs(), nil) + out, _ := ext.ApplyModifiers(render.ComponentNameNode, entCtx(), newObjs(), nil) ds, _ := extensions.FindObject[*appsv1.DaemonSet](out, common.NodeDaemonSetName) c := nodeContainer(ds) @@ -115,13 +110,13 @@ var _ = Describe("node enterprise modifier", func() { ctx := entCtx() ctx.FelixConfiguration = &v3.FelixConfiguration{Spec: v3.FelixConfigurationSpec{PrometheusReporterPort: &reporter}} - out, _ := extensions.ApplyModifiers(render.ComponentNameNode, ctx, newObjs(), nil) + out, _ := ext.ApplyModifiers(render.ComponentNameNode, ctx, newObjs(), nil) ds, _ := extensions.FindObject[*appsv1.DaemonSet](out, common.NodeDaemonSetName) Expect(nodeContainer(ds).Env).To(ContainElement(corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERPORT", Value: "7081"})) }) It("appends the BGP metrics readiness check when the bird check is present", func() { - out, _ := extensions.ApplyModifiers(render.ComponentNameNode, entCtx(), newObjs(), nil) + out, _ := ext.ApplyModifiers(render.ComponentNameNode, entCtx(), newObjs(), nil) ds, _ := extensions.FindObject[*appsv1.DaemonSet](out, common.NodeDaemonSetName) Expect(nodeContainer(ds).ReadinessProbe.Exec.Command).To(ContainElement("--bgp-metrics-ready")) }) @@ -131,7 +126,7 @@ var _ = Describe("node enterprise modifier", func() { ds := objs[2].(*appsv1.DaemonSet) ds.Spec.Template.Spec.Containers[0].ReadinessProbe.Exec.Command = []string{"/bin/calico-node", "--felix-ready"} - out, _ := extensions.ApplyModifiers(render.ComponentNameNode, entCtx(), objs, nil) + out, _ := ext.ApplyModifiers(render.ComponentNameNode, entCtx(), objs, nil) got, _ := extensions.FindObject[*appsv1.DaemonSet](out, common.NodeDaemonSetName) Expect(nodeContainer(got).ReadinessProbe.Exec.Command).NotTo(ContainElement("--bgp-metrics-ready")) }) @@ -141,7 +136,7 @@ var _ = Describe("node enterprise modifier", func() { ctx := entCtx() ctx.Installation.CalicoNetwork = &operatorv1.CalicoNetworkSpec{MultiInterfaceMode: &mode} - out, _ := extensions.ApplyModifiers(render.ComponentNameNode, ctx, newObjs(), nil) + out, _ := ext.ApplyModifiers(render.ComponentNameNode, ctx, newObjs(), nil) ds, _ := extensions.FindObject[*appsv1.DaemonSet](out, common.NodeDaemonSetName) want := corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: mode.Value()} @@ -150,7 +145,7 @@ var _ = Describe("node enterprise modifier", func() { }) It("appends the node metrics service", func() { - out, _ := extensions.ApplyModifiers(render.ComponentNameNode, entCtx(), newObjs(), nil) + out, _ := ext.ApplyModifiers(render.ComponentNameNode, entCtx(), newObjs(), nil) svc, ok := extensions.FindObject[*corev1.Service](out, render.CalicoNodeMetricsService) Expect(ok).To(BeTrue()) Expect(svc.Spec.Ports).To(HaveLen(2)) @@ -169,7 +164,7 @@ var _ = Describe("node enterprise modifier", func() { PrometheusMetricsEnabled: &enabled, }} - out, _ := extensions.ApplyModifiers(render.ComponentNameNode, ctx, newObjs(), nil) + out, _ := ext.ApplyModifiers(render.ComponentNameNode, ctx, newObjs(), nil) svc, _ := extensions.FindObject[*corev1.Service](out, render.CalicoNodeMetricsService) Expect(svc.Spec.Ports).To(HaveLen(3)) Expect(svc.Spec.Ports[0].Port).To(Equal(int32(7081))) @@ -179,7 +174,7 @@ var _ = Describe("node enterprise modifier", func() { It("is a no-op for the Calico variant", func() { ctx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}} - out, _ := extensions.ApplyModifiers(render.ComponentNameNode, ctx, newObjs(), nil) + out, _ := ext.ApplyModifiers(render.ComponentNameNode, ctx, newObjs(), nil) _, ok := extensions.FindObject[*corev1.Service](out, render.CalicoNodeMetricsService) Expect(ok).To(BeFalse()) @@ -188,7 +183,7 @@ var _ = Describe("node enterprise modifier", func() { }) It("does not panic on a zero RenderContext", func() { - out, _ := extensions.ApplyModifiers(render.ComponentNameNode, extensions.RenderContext{}, newObjs(), nil) + out, _ := ext.ApplyModifiers(render.ComponentNameNode, extensions.RenderContext{}, newObjs(), nil) _, ok := extensions.FindObject[*corev1.Service](out, render.CalicoNodeMetricsService) Expect(ok).To(BeFalse()) }) diff --git a/pkg/enterprise/register.go b/pkg/enterprise/register.go index 8d6bdc0cd8..d1f81451b1 100644 --- a/pkg/enterprise/register.go +++ b/pkg/enterprise/register.go @@ -14,14 +14,19 @@ package enterprise -// Register wires all in-repo enterprise modifiers and controller extensions -// into the operator registries. Called once at process startup. After the -// monorepo split this is what calico-private's main will do instead. -func Register() { - registerTypha() - registerNode() - registerWindows() - registerGuardian() - registerInstallation() - registerAPIServer() +import "github.com/tigera/operator/pkg/extensions" + +// New builds the extension Set for the in-repo Calico Enterprise variant: every +// component modifier, image override, and the installation setup. The operator +// is handed this Set at startup (the core operator is handed none). After the +// monorepo split this is what calico-private's main will construct instead. +func New() *extensions.Set { + s := extensions.NewSet() + registerTypha(s) + registerNode(s) + registerWindows(s) + registerGuardian(s) + registerInstallation(s) + registerAPIServer(s) + return s } diff --git a/pkg/enterprise/typha.go b/pkg/enterprise/typha.go index 38733c871c..4571450365 100644 --- a/pkg/enterprise/typha.go +++ b/pkg/enterprise/typha.go @@ -25,8 +25,8 @@ import ( "github.com/tigera/operator/pkg/render" ) -func registerTypha() { - extensions.Register(operatorv1.CalicoEnterprise, render.ComponentNameTypha, extensions.Extension{ +func registerTypha(s *extensions.Set) { + s.Register(operatorv1.CalicoEnterprise, render.ComponentNameTypha, extensions.Extension{ Modify: modifyTypha, }) } diff --git a/pkg/enterprise/typha_test.go b/pkg/enterprise/typha_test.go index 2ebb853f8c..2b1211c4ec 100644 --- a/pkg/enterprise/typha_test.go +++ b/pkg/enterprise/typha_test.go @@ -24,16 +24,11 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" operatorv1 "github.com/tigera/operator/api/v1" - "github.com/tigera/operator/pkg/enterprise" "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" ) var _ = Describe("typha enterprise modifier", func() { - BeforeEach(func() { enterprise.Register() }) - AfterEach(func() { - extensions.ResetForTest() - }) multiMode := operatorv1.MultiInterfaceModeMultus @@ -54,7 +49,7 @@ var _ = Describe("typha enterprise modifier", func() { Variant: operatorv1.CalicoEnterprise, CalicoNetwork: &operatorv1.CalicoNetworkSpec{MultiInterfaceMode: &multiMode}, }} - out, _ := extensions.ApplyModifiers(render.ComponentNameTypha, ctx, newObjs(), nil) + out, _ := ext.ApplyModifiers(render.ComponentNameTypha, ctx, newObjs(), nil) role := out[0].(*rbacv1.ClusterRole) Expect(role.Rules).To(ContainElement(HaveField("Resources", ContainElement("licensekeys")))) @@ -74,14 +69,14 @@ var _ = Describe("typha enterprise modifier", func() { Variant: operatorv1.Calico, CalicoNetwork: &operatorv1.CalicoNetworkSpec{MultiInterfaceMode: &multiMode}, }} - out, _ := extensions.ApplyModifiers(render.ComponentNameTypha, ctx, newObjs(), nil) + out, _ := ext.ApplyModifiers(render.ComponentNameTypha, ctx, newObjs(), nil) Expect(out[0].(*rbacv1.ClusterRole).Rules).To(BeEmpty()) dep := out[1].(*appsv1.Deployment) Expect(dep.Spec.Template.Spec.Containers[0].Env).To(BeEmpty()) }) It("does not panic on a zero Context (nil Installation)", func() { - out, _ := extensions.ApplyModifiers(render.ComponentNameTypha, extensions.RenderContext{}, newObjs(), nil) + out, _ := ext.ApplyModifiers(render.ComponentNameTypha, extensions.RenderContext{}, newObjs(), nil) Expect(out[0].(*rbacv1.ClusterRole).Rules).To(BeEmpty()) }) }) diff --git a/pkg/enterprise/windows.go b/pkg/enterprise/windows.go index 8a3e086835..f4674323b3 100644 --- a/pkg/enterprise/windows.go +++ b/pkg/enterprise/windows.go @@ -36,14 +36,14 @@ import ( // felix env and node volume mounts, so they receive the same enterprise layering. var windowsNodeContainers = map[string]bool{"felix": true, "node": true, "confd": true} -func registerWindows() { - extensions.Register(operatorv1.CalicoEnterprise, render.ComponentNameWindowsNodeImg, extensions.Extension{ +func registerWindows(s *extensions.Set) { + s.Register(operatorv1.CalicoEnterprise, render.ComponentNameWindowsNodeImg, extensions.Extension{ Image: func(*operatorv1.InstallationSpec) components.Component { return components.ComponentTigeraNodeWindows }, }) - extensions.Register(operatorv1.CalicoEnterprise, render.ComponentNameWindowsCNIImg, extensions.Extension{ + s.Register(operatorv1.CalicoEnterprise, render.ComponentNameWindowsCNIImg, extensions.Extension{ Image: func(*operatorv1.InstallationSpec) components.Component { return components.ComponentTigeraCNIWindows }, }) - extensions.Register(operatorv1.CalicoEnterprise, render.ComponentNameWindows, extensions.Extension{ + s.Register(operatorv1.CalicoEnterprise, render.ComponentNameWindows, extensions.Extension{ Modify: modifyWindows, }) } diff --git a/pkg/enterprise/windows_test.go b/pkg/enterprise/windows_test.go index 18668c9079..9bc08e1d52 100644 --- a/pkg/enterprise/windows_test.go +++ b/pkg/enterprise/windows_test.go @@ -30,33 +30,28 @@ import ( "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/controller/certificatemanager" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" - "github.com/tigera/operator/pkg/enterprise" "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" "github.com/tigera/operator/pkg/tls/certificatemanagement" ) var _ = Describe("windows enterprise image override", func() { - BeforeEach(func() { enterprise.Register() }) - AfterEach(func() { extensions.ResetForTest() }) ent := &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise} calico := &operatorv1.InstallationSpec{Variant: operatorv1.Calico} It("selects the enterprise windows images for the enterprise variant", func() { - Expect(extensions.ResolveImage(render.ComponentNameWindowsNodeImg, components.ComponentCalicoNodeWindows, ent)).To(Equal(components.ComponentTigeraNodeWindows)) - Expect(extensions.ResolveImage(render.ComponentNameWindowsCNIImg, components.ComponentCalicoCNIWindows, ent)).To(Equal(components.ComponentTigeraCNIWindows)) + Expect(ext.ResolveImage(render.ComponentNameWindowsNodeImg, components.ComponentCalicoNodeWindows, ent)).To(Equal(components.ComponentTigeraNodeWindows)) + Expect(ext.ResolveImage(render.ComponentNameWindowsCNIImg, components.ComponentCalicoCNIWindows, ent)).To(Equal(components.ComponentTigeraCNIWindows)) }) It("leaves the defaults in place for the Calico variant", func() { - Expect(extensions.ResolveImage(render.ComponentNameWindowsNodeImg, components.ComponentCalicoNodeWindows, calico)).To(Equal(components.ComponentCalicoNodeWindows)) - Expect(extensions.ResolveImage(render.ComponentNameWindowsCNIImg, components.ComponentCalicoCNIWindows, calico)).To(Equal(components.ComponentCalicoCNIWindows)) + Expect(ext.ResolveImage(render.ComponentNameWindowsNodeImg, components.ComponentCalicoNodeWindows, calico)).To(Equal(components.ComponentCalicoNodeWindows)) + Expect(ext.ResolveImage(render.ComponentNameWindowsCNIImg, components.ComponentCalicoCNIWindows, calico)).To(Equal(components.ComponentCalicoCNIWindows)) }) }) var _ = Describe("windows enterprise modifier", func() { - BeforeEach(func() { enterprise.Register() }) - AfterEach(func() { extensions.ResetForTest() }) // newObjs returns a windows daemonset with the node containers and the OSS // cni-log-dir mount the modifier swaps out. @@ -102,7 +97,7 @@ var _ = Describe("windows enterprise modifier", func() { } It("appends the node-metrics service", func() { - out, _ := extensions.ApplyModifiers(render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone, nil, nil), newObjs(), nil) + out, _ := ext.ApplyModifiers(render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone, nil, nil), newObjs(), nil) svc, ok := extensions.FindObject[*corev1.Service](out, render.WindowsNodeMetricsService) Expect(ok).To(BeTrue()) Expect(svc.Namespace).To(Equal(common.CalicoNamespace)) @@ -110,7 +105,7 @@ var _ = Describe("windows enterprise modifier", func() { }) It("swaps the cni log mount for the calico log volume and adds enterprise env", func() { - out, _ := extensions.ApplyModifiers(render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone, nil, nil), newObjs(), nil) + out, _ := ext.ApplyModifiers(render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone, nil, nil), newObjs(), nil) d := ds(out) Expect(d.Spec.Template.Spec.Volumes).To(ContainElement(HaveField("Name", "var-log-calico"))) @@ -127,7 +122,7 @@ var _ = Describe("windows enterprise modifier", func() { }) It("sets the trusted DNS server on openshift", func() { - out, _ := extensions.ApplyModifiers(render.ComponentNameWindows, ctxFor(operatorv1.ProviderOpenShift, nil, nil), newObjs(), nil) + out, _ := ext.ApplyModifiers(render.ComponentNameWindows, ctxFor(operatorv1.ProviderOpenShift, nil, nil), newObjs(), nil) Expect(container(ds(out), "node").Env).To(ContainElement(corev1.EnvVar{Name: "FELIX_DNSTRUSTEDSERVERS", Value: "k8s-service:openshift-dns/dns-default"})) }) @@ -141,7 +136,7 @@ var _ = Describe("windows enterprise modifier", func() { Expect(err).NotTo(HaveOccurred()) bundle := cm.CreateTrustedBundle() - out, _ := extensions.ApplyModifiers(render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone, tls, bundle), newObjs(), nil) + out, _ := ext.ApplyModifiers(render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone, tls, bundle), newObjs(), nil) d := ds(out) Expect(d.Spec.Template.Spec.Volumes).To(ContainElement(tls.Volume())) @@ -152,7 +147,7 @@ var _ = Describe("windows enterprise modifier", func() { It("does nothing for the Calico variant", func() { ctx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}} - out, _ := extensions.ApplyModifiers(render.ComponentNameWindows, ctx, newObjs(), nil) + out, _ := ext.ApplyModifiers(render.ComponentNameWindows, ctx, newObjs(), nil) _, ok := extensions.FindObject[*corev1.Service](out, render.WindowsNodeMetricsService) Expect(ok).To(BeFalse()) Expect(ds(out).Spec.Template.Spec.Volumes).To(BeEmpty()) diff --git a/pkg/extensions/extension.go b/pkg/extensions/extension.go index 25a43feb13..346e7347f2 100644 --- a/pkg/extensions/extension.go +++ b/pkg/extensions/extension.go @@ -18,7 +18,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" operatorv1 "github.com/tigera/operator/api/v1" - "github.com/tigera/operator/pkg/imageoverride" ) // Extension is everything a variant layers onto one render component. Every @@ -48,30 +47,16 @@ type modifierKey struct { component string } -var modifiers = map[modifierKey]Modifier{} - -// Register installs e as the extension for the named component under the given -// variant. A (variant, component) pair has at most one extension; registration -// replaces any prior one, so it is idempotent and safe to call more than once. -// The image override lives in the imageoverride leaf (so the render package can -// resolve it without an import cycle); the modifier lives here. -func Register(variant operatorv1.ProductVariant, component string, e Extension) { - if e.Image != nil { - imageoverride.Register(variant, component, e.Image) - } - if e.Modify != nil { - modifiers[modifierKey{variant, component}] = e.Modify - } -} - // ApplyModifiers runs the modifier registered for the named component and the // installation's variant over the create and delete lists, returning them -// unchanged when none is registered (or when no installation is set). -func ApplyModifiers(component string, ctx RenderContext, create, delete []client.Object) ([]client.Object, []client.Object) { - if ctx.Installation == nil { +// unchanged when none is registered (or when no installation is set). Safe to +// call on a nil Set, which is a no-op - the core operator registers no +// modifiers. +func (s *Set) ApplyModifiers(component string, ctx RenderContext, create, delete []client.Object) ([]client.Object, []client.Object) { + if s == nil || ctx.Installation == nil { return create, delete } - if fn, ok := modifiers[modifierKey{ctx.Installation.Variant, component}]; ok { + if fn, ok := s.modifiers[modifierKey{ctx.Installation.Variant, component}]; ok { create, delete = fn(ctx, create, delete) } return create, delete @@ -87,11 +72,3 @@ func FindObject[T client.Object](objs []client.Object, name string) (T, bool) { } return zero, false } - -// ResetForTest clears every registry: modifiers, image overrides, and variant -// setups. Test-only. -func ResetForTest() { - modifiers = map[modifierKey]Modifier{} - setups = map[operatorv1.ProductVariant]Setup{} - imageoverride.ResetForTest() -} diff --git a/pkg/extensions/extension_test.go b/pkg/extensions/extension_test.go index 2c08569355..b02ec3d24d 100644 --- a/pkg/extensions/extension_test.go +++ b/pkg/extensions/extension_test.go @@ -26,14 +26,15 @@ import ( ) var _ = Describe("extension registry", func() { - AfterEach(func() { - extensions.ResetForTest() + var s *extensions.Set + BeforeEach(func() { + s = extensions.NewSet() }) entCtx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}} It("applies a registered modifier to the matching component and variant", func() { - extensions.Register(operatorv1.CalicoEnterprise, "test", extensions.Extension{ + s.Register(operatorv1.CalicoEnterprise, "test", extensions.Extension{ Modify: func(ctx extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { cm, ok := extensions.FindObject[*corev1.ConfigMap](objs, "cm") Expect(ok).To(BeTrue()) @@ -43,7 +44,7 @@ var _ = Describe("extension registry", func() { }) in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} - out, _ := extensions.ApplyModifiers("test", entCtx, in, nil) + out, _ := s.ApplyModifiers("test", entCtx, in, nil) Expect(out).To(HaveLen(2)) cm := out[0].(*corev1.ConfigMap) @@ -52,14 +53,14 @@ var _ = Describe("extension registry", func() { }) It("lets a modifier append to the delete list", func() { - extensions.Register(operatorv1.CalicoEnterprise, "test", extensions.Extension{ + s.Register(operatorv1.CalicoEnterprise, "test", extensions.Extension{ Modify: func(_ extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { return objs, append(del, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "stale"}}) }, }) in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} - out, del := extensions.ApplyModifiers("test", entCtx, in, nil) + out, del := s.ApplyModifiers("test", entCtx, in, nil) Expect(out).To(Equal(in)) Expect(del).To(HaveLen(1)) Expect(del[0].GetName()).To(Equal("stale")) @@ -67,12 +68,12 @@ var _ = Describe("extension registry", func() { It("returns objects unchanged when no modifier is registered", func() { in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} - out, _ := extensions.ApplyModifiers("unregistered", entCtx, in, nil) + out, _ := s.ApplyModifiers("unregistered", entCtx, in, nil) Expect(out).To(Equal(in)) }) It("does not apply a modifier registered for a different variant", func() { - extensions.Register(operatorv1.CalicoEnterprise, "test", extensions.Extension{ + s.Register(operatorv1.CalicoEnterprise, "test", extensions.Extension{ Modify: func(_ extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { return append(objs, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "extra"}}), del }, @@ -80,13 +81,13 @@ var _ = Describe("extension registry", func() { calicoCtx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}} in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} - out, _ := extensions.ApplyModifiers("test", calicoCtx, in, nil) + out, _ := s.ApplyModifiers("test", calicoCtx, in, nil) Expect(out).To(Equal(in)) }) It("returns objects unchanged when no installation is set", func() { in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} - out, _ := extensions.ApplyModifiers("test", extensions.RenderContext{}, in, nil) + out, _ := s.ApplyModifiers("test", extensions.RenderContext{}, in, nil) Expect(out).To(Equal(in)) }) @@ -98,10 +99,10 @@ var _ = Describe("extension registry", func() { }, } } - extensions.Register(operatorv1.CalicoEnterprise, "test", add("first")) - extensions.Register(operatorv1.CalicoEnterprise, "test", add("second")) + s.Register(operatorv1.CalicoEnterprise, "test", add("first")) + s.Register(operatorv1.CalicoEnterprise, "test", add("second")) - out, _ := extensions.ApplyModifiers("test", entCtx, nil, nil) + out, _ := s.ApplyModifiers("test", entCtx, nil, nil) Expect(out).To(HaveLen(1)) Expect(out[0].GetName()).To(Equal("second")) }) diff --git a/pkg/extensions/image.go b/pkg/extensions/image.go index 477c89deef..a551d1e76f 100644 --- a/pkg/extensions/image.go +++ b/pkg/extensions/image.go @@ -15,8 +15,6 @@ package extensions import ( - operatorv1 "github.com/tigera/operator/api/v1" - "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/imageoverride" ) @@ -24,10 +22,3 @@ import ( // the Image field of an Extension. An override runs only for the variant it was // registered under, so it need not re-check the variant. type ImageOverride = imageoverride.Override - -// ResolveImage returns the override registered for key if it applies to in, -// otherwise def. The render package resolves images through the imageoverride -// leaf directly; this is the same lookup for callers already inside extensions. -func ResolveImage(key string, def components.Component, in *operatorv1.InstallationSpec) components.Component { - return imageoverride.Resolve(key, def, in) -} diff --git a/pkg/extensions/image_test.go b/pkg/extensions/image_test.go index 25926c1a4f..0581d8cc08 100644 --- a/pkg/extensions/image_test.go +++ b/pkg/extensions/image_test.go @@ -24,29 +24,23 @@ import ( ) var _ = Describe("image overrides", func() { - AfterEach(func() { - extensions.ResetForTest() - }) - - It("uses the override registered for the installation variant", func() { - extensions.Register(operatorv1.CalicoEnterprise, "node", extensions.Extension{ + var s *extensions.Set + BeforeEach(func() { + s = extensions.NewSet() + s.Register(operatorv1.CalicoEnterprise, "node", extensions.Extension{ Image: func(in *operatorv1.InstallationSpec) components.Component { return components.ComponentTigeraNode }, }) + }) + It("uses the override registered for the installation variant", func() { ent := &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise} - Expect(extensions.ResolveImage("node", components.ComponentCalicoNode, ent)).To(Equal(components.ComponentTigeraNode)) + Expect(s.ResolveImage("node", components.ComponentCalicoNode, ent)).To(Equal(components.ComponentTigeraNode)) }) It("falls back to the default for a variant with no override", func() { - extensions.Register(operatorv1.CalicoEnterprise, "node", extensions.Extension{ - Image: func(in *operatorv1.InstallationSpec) components.Component { - return components.ComponentTigeraNode - }, - }) - calico := &operatorv1.InstallationSpec{Variant: operatorv1.Calico} - Expect(extensions.ResolveImage("node", components.ComponentCalicoNode, calico)).To(Equal(components.ComponentCalicoNode)) + Expect(s.ResolveImage("node", components.ComponentCalicoNode, calico)).To(Equal(components.ComponentCalicoNode)) }) }) diff --git a/pkg/extensions/set.go b/pkg/extensions/set.go new file mode 100644 index 0000000000..b638622750 --- /dev/null +++ b/pkg/extensions/set.go @@ -0,0 +1,82 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extensions + +import ( + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/imageoverride" +) + +// Set is the collection of variant extensions the operator runs with: the +// per-variant setups, the per-component modifiers, and the image overrides. The +// core operator runs with a nil/empty Set; an extension build (Calico +// Enterprise) constructs a populated one and hands it in through +// options.ControllerOptions. This replaces what used to be package-level +// registries, so nothing is wired by import side effect. +// +// The zero value is not usable; build one with NewSet. The methods that read it +// (BuildContext, ApplyModifiers, ResolveImage, Images) are nil-safe so the core +// operator can pass a nil Set and get base behavior. +type Set struct { + setups map[operatorv1.ProductVariant]Setup + modifiers map[modifierKey]Modifier + images *imageoverride.Overrides +} + +// NewSet returns an empty Set ready to register extensions into. +func NewSet() *Set { + return &Set{ + setups: map[operatorv1.ProductVariant]Setup{}, + modifiers: map[modifierKey]Modifier{}, + images: imageoverride.New(), + } +} + +// Register installs e as the extension for the named component under the given +// variant. A (variant, component) pair has at most one extension; registration +// replaces any prior one. The image override and the modifier are stored +// separately, so a component can set either field or both. +func (s *Set) Register(variant operatorv1.ProductVariant, component string, e Extension) { + if e.Image != nil { + s.images.Register(variant, component, e.Image) + } + if e.Modify != nil { + s.modifiers[modifierKey{variant, component}] = e.Modify + } +} + +// RegisterSetup installs setup as the controller-side setup phase for the given +// variant. Registration replaces any prior setup for that variant. +func (s *Set) RegisterSetup(variant operatorv1.ProductVariant, setup Setup) { + s.setups[variant] = setup +} + +// Images returns the image overrides. The render package resolves a component's +// image through these directly (the imageoverride leaf, so render need not +// import extensions). Safe to call on a nil Set, which returns nil overrides +// that resolve to the default image. +func (s *Set) Images() *imageoverride.Overrides { + if s == nil { + return nil + } + return s.images +} + +// ResolveImage resolves key for the installation through the image overrides, +// returning def when no override applies. Safe to call on a nil Set. +func (s *Set) ResolveImage(key string, def components.Component, in *operatorv1.InstallationSpec) components.Component { + return s.Images().Resolve(key, def, in) +} diff --git a/pkg/extensions/setup.go b/pkg/extensions/setup.go index a6272b006c..2eb0ef1b6b 100644 --- a/pkg/extensions/setup.go +++ b/pkg/extensions/setup.go @@ -64,22 +64,14 @@ func BaseRenderContext(in Inputs) RenderContext { } } -var setups = map[operatorv1.ProductVariant]Setup{} - -// RegisterSetup installs s as the setup for the given variant. Registration -// replaces any prior setup, so it is safe to call more than once. Variants -// without a registered setup get the base render context. -func RegisterSetup(variant operatorv1.ProductVariant, s Setup) { - setups[variant] = s -} - // BuildContext runs the setup registered for the installation variant and // returns its RenderContext, or the base render context when the variant has no -// setup. -func BuildContext(in Inputs) (RenderContext, error) { - if in.Installation != nil { - if s, ok := setups[in.Installation.Variant]; ok { - return s(in) +// setup. Safe to call on a nil Set, which always returns the base context - the +// core operator registers no setups. +func (s *Set) BuildContext(in Inputs) (RenderContext, error) { + if s != nil && in.Installation != nil { + if setup, ok := s.setups[in.Installation.Variant]; ok { + return setup(in) } } return BaseRenderContext(in), nil diff --git a/pkg/extensions/setup_test.go b/pkg/extensions/setup_test.go index 1ba42e5567..9ee58a6648 100644 --- a/pkg/extensions/setup_test.go +++ b/pkg/extensions/setup_test.go @@ -25,11 +25,14 @@ import ( ) var _ = Describe("variant setup", func() { - AfterEach(func() { extensions.ResetForTest() }) + var s *extensions.Set + BeforeEach(func() { + s = extensions.NewSet() + }) It("returns the base render context when the variant has no setup", func() { install := &operatorv1.InstallationSpec{Variant: operatorv1.Calico} - rc, err := extensions.BuildContext(extensions.Inputs{ + rc, err := s.BuildContext(extensions.Inputs{ Installation: install, ClusterDomain: "cluster.local", }) @@ -40,15 +43,15 @@ var _ = Describe("variant setup", func() { }) It("uses the setup registered for the installation variant", func() { - extensions.RegisterSetup(operatorv1.CalicoEnterprise, fakeSetup(nil)) - rc, err := extensions.BuildContext(enterpriseInputs()) + s.RegisterSetup(operatorv1.CalicoEnterprise, fakeSetup(nil)) + rc, err := s.BuildContext(enterpriseInputs()) Expect(err).NotTo(HaveOccurred()) Expect(rc.ClusterDomain).To(Equal("from-fake")) }) It("ignores a setup registered for a different variant", func() { - extensions.RegisterSetup(operatorv1.CalicoEnterprise, fakeSetup(nil)) - rc, err := extensions.BuildContext(extensions.Inputs{ + s.RegisterSetup(operatorv1.CalicoEnterprise, fakeSetup(nil)) + rc, err := s.BuildContext(extensions.Inputs{ Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}, ClusterDomain: "real", }) @@ -57,17 +60,16 @@ var _ = Describe("variant setup", func() { }) It("surfaces the setup error", func() { - extensions.RegisterSetup(operatorv1.CalicoEnterprise, fakeSetup(errors.New("boom"))) - _, err := extensions.BuildContext(enterpriseInputs()) + s.RegisterSetup(operatorv1.CalicoEnterprise, fakeSetup(errors.New("boom"))) + _, err := s.BuildContext(enterpriseInputs()) Expect(err).To(MatchError("boom")) }) - It("restores the base context on reset", func() { - extensions.RegisterSetup(operatorv1.CalicoEnterprise, fakeSetup(nil)) - extensions.ResetForTest() + It("returns the base context for a nil Set", func() { + var nilSet *extensions.Set in := enterpriseInputs() in.ClusterDomain = "real" - rc, err := extensions.BuildContext(in) + rc, err := nilSet.BuildContext(in) Expect(err).NotTo(HaveOccurred()) Expect(rc.ClusterDomain).To(Equal("real")) }) diff --git a/pkg/imageoverride/imageoverride.go b/pkg/imageoverride/imageoverride.go index c3bc471bc2..5c981c0c07 100644 --- a/pkg/imageoverride/imageoverride.go +++ b/pkg/imageoverride/imageoverride.go @@ -13,8 +13,8 @@ // limitations under the License. // Package imageoverride is a leaf package (no render/operator dependencies) -// that holds the image override registry. Both pkg/operator and pkg/render -// import it to avoid the render→operator→render import cycle. +// that holds the image override table. The render package imports it to resolve +// a component's image without depending on pkg/extensions, which would cycle. package imageoverride import ( @@ -30,27 +30,33 @@ type overrideKey struct { key string } -var registry = map[overrideKey]Override{} +// Overrides maps a component (keyed by variant) to the image it should resolve +// to, letting a variant swap a component's image without the render package +// branching on variant. The render component holds one and resolves through it. +type Overrides struct { + m map[overrideKey]Override +} + +// New returns an empty Overrides. +func New() *Overrides { + return &Overrides{m: map[overrideKey]Override{}} +} // Register stores fn under key for the given variant. The key is the render // component's image identifier (e.g. "node"). -func Register(variant operatorv1.ProductVariant, key string, fn Override) { - registry[overrideKey{variant, key}] = fn +func (o *Overrides) Register(variant operatorv1.ProductVariant, key string, fn Override) { + o.m[overrideKey{variant, key}] = fn } // Resolve returns the override registered for key under the installation's -// variant, otherwise def. -func Resolve(key string, def components.Component, in *operatorv1.InstallationSpec) components.Component { - if in == nil { +// variant, otherwise def. It is safe to call on a nil *Overrides (the core +// operator hands render no overrides), which always returns def. +func (o *Overrides) Resolve(key string, def components.Component, in *operatorv1.InstallationSpec) components.Component { + if o == nil || in == nil { return def } - if fn, ok := registry[overrideKey{in.Variant, key}]; ok { + if fn, ok := o.m[overrideKey{in.Variant, key}]; ok { return fn(in) } return def } - -// ResetForTest clears the registry. Test-only. -func ResetForTest() { - registry = map[overrideKey]Override{} -} diff --git a/pkg/render/apiserver_test.go b/pkg/render/apiserver_test.go index 3645e16f33..a81ebb6e01 100644 --- a/pkg/render/apiserver_test.go +++ b/pkg/render/apiserver_test.go @@ -73,7 +73,7 @@ func apiServerObjects(c render.Component) ([]client.Object, []client.Object) { rc.Installation = ec.Config.Installation rc.Component = ec } - return extensions.ApplyModifiers(render.ComponentNameAPIServer, rc, create, del) + return ext.ApplyModifiers(render.ComponentNameAPIServer, rc, create, del) } var _ = Describe("API server rendering tests (Calico Enterprise)", func() { diff --git a/pkg/render/enterprise_setup_test.go b/pkg/render/enterprise_setup_test.go index 13766f7035..e896366124 100644 --- a/pkg/render/enterprise_setup_test.go +++ b/pkg/render/enterprise_setup_test.go @@ -15,20 +15,13 @@ package render_test import ( - . "github.com/onsi/ginkgo/v2" - "github.com/tigera/operator/pkg/enterprise" + "github.com/tigera/operator/pkg/extensions" ) -// Register the enterprise extensions once for the whole render suite. This wires -// two things the suite relies on: -// - the image override, which the Objects()-level render tests pick up through -// ResolveImages (e.g. the enterprise node image), and -// - the modifiers, which node_enterprise_test.go applies explicitly to real -// render output to check they still match it. -// -// The plain Objects()-level tests do not run modifiers - those only run at the -// componentHandler - so registering here does not change their output. -var _ = BeforeSuite(func() { - enterprise.Register() -}) +// ext is the enterprise extension Set the render suite tests against. The +// Objects()-level image tests pass ext.Images() into the node/windows configs to +// pick up the enterprise images, and the enterprise modifier tests apply ext's +// modifiers explicitly to real render output to check they still match it. It is +// immutable once built and specs only read it, so a single instance is safe. +var ext *extensions.Set = enterprise.New() diff --git a/pkg/render/guardian_test.go b/pkg/render/guardian_test.go index b28e77f044..33c04099fd 100644 --- a/pkg/render/guardian_test.go +++ b/pkg/render/guardian_test.go @@ -54,7 +54,7 @@ func guardianObjects(cfg *render.GuardianConfiguration) []client.Object { if p, ok := g.(render.ExtensionContextProvider); ok { rc.Component = p.ExtensionContext() } - out, _ := extensions.ApplyModifiers(render.GuardianName, rc, objs, nil) + out, _ := ext.ApplyModifiers(render.GuardianName, rc, objs, nil) return out } @@ -116,7 +116,7 @@ var _ = Describe("Rendering tests", func() { if p, ok := g.(render.ExtensionContextProvider); ok { rc.Component = p.ExtensionContext() } - resources, _ = extensions.ApplyModifiers(render.GuardianName, rc, resources, nil) + resources, _ = ext.ApplyModifiers(render.GuardianName, rc, resources, nil) } BeforeEach(func() { @@ -352,7 +352,7 @@ var _ = Describe("Rendering tests", func() { if p, ok := g.(render.ExtensionContextProvider); ok { rc.Component = p.ExtensionContext() } - resources, _ = extensions.ApplyModifiers(render.ComponentNameGuardianPolicy, rc, objs, nil) + resources, _ = ext.ApplyModifiers(render.ComponentNameGuardianPolicy, rc, objs, nil) } Context("policy rendering based on variant and IncludeEgressNetworkPolicy", func() { diff --git a/pkg/render/node.go b/pkg/render/node.go index ddcc43e897..4077e43db2 100644 --- a/pkg/render/node.go +++ b/pkg/render/node.go @@ -145,6 +145,11 @@ type NodeConfiguration struct { BindMode string V3CRDs bool + + // ImageOverrides lets a variant swap the node and cni-plugins images. The + // controller wires in the operator's image overrides; nil resolves to the + // core images. + ImageOverrides *imageoverride.Overrides } // Node creates the node daemonset and other resources for the daemonset to operate normally. @@ -179,10 +184,10 @@ func (c *nodeComponent) ResolveImages(is *operatorv1.ImageSet) error { } c.calicoImage = appendIfErr(components.GetReference(components.CombinedCalicoImage(c.cfg.Installation), reg, path, prefix, is)) - nodeImage := imageoverride.Resolve(ComponentNameNode, components.ComponentCalicoNode, c.cfg.Installation) + nodeImage := c.cfg.ImageOverrides.Resolve(ComponentNameNode, components.ComponentCalicoNode, c.cfg.Installation) c.nodeImage = appendIfErr(components.GetReference(nodeImage, reg, path, prefix, is)) if c.installUpstreamPlugins() { - cniPluginsImage := imageoverride.Resolve(ComponentNameCNIPlugins, components.ComponentCalicoCNIPlugins, c.cfg.Installation) + cniPluginsImage := c.cfg.ImageOverrides.Resolve(ComponentNameCNIPlugins, components.ComponentCalicoCNIPlugins, c.cfg.Installation) c.cniPluginsImage = appendIfErr(components.GetReference(cniPluginsImage, reg, path, prefix, is)) } diff --git a/pkg/render/node_enterprise_test.go b/pkg/render/node_enterprise_test.go index 4b9b6a2f82..7b11bbf6eb 100644 --- a/pkg/render/node_enterprise_test.go +++ b/pkg/render/node_enterprise_test.go @@ -110,7 +110,7 @@ var _ = Describe("node enterprise modifier integration", func() { comp := render.Node(cfg) Expect(comp.ResolveImages(nil)).NotTo(HaveOccurred()) objs, _ := comp.Objects() - out, _ := extensions.ApplyModifiers(render.ComponentNameNode, renderCtx, objs, nil) + out, _ := ext.ApplyModifiers(render.ComponentNameNode, renderCtx, objs, nil) return out } @@ -167,7 +167,7 @@ var _ = Describe("node enterprise modifier integration", func() { }) Expect(comp.ResolveImages(nil)).NotTo(HaveOccurred()) objs, _ := comp.Objects() - objs, _ = extensions.ApplyModifiers(render.ComponentNameTypha, renderCtx, objs, nil) + objs, _ = ext.ApplyModifiers(render.ComponentNameTypha, renderCtx, objs, nil) role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, "calico-typha") Expect(ok).To(BeTrue()) diff --git a/pkg/render/node_test.go b/pkg/render/node_test.go index 3b2b487d53..b052be0ef4 100644 --- a/pkg/render/node_test.go +++ b/pkg/render/node_test.go @@ -140,6 +140,7 @@ var _ = Describe("Node rendering tests", func() { ClusterDomain: defaultClusterDomain, FelixHealthPort: 9099, IPPools: defaultInstance.CalicoNetwork.IPPools, + ImageOverrides: ext.Images(), } }) diff --git a/pkg/render/windows.go b/pkg/render/windows.go index f27120979b..7392094458 100644 --- a/pkg/render/windows.go +++ b/pkg/render/windows.go @@ -57,6 +57,11 @@ type WindowsConfiguration struct { PrometheusServerTLS certificatemanagement.KeyPairInterface NodeReporterMetricsPort int VXLANVNI int + + // ImageOverrides lets a variant swap the windows node and CNI images. The + // controller wires in the operator's image overrides; nil resolves to the + // core images. + ImageOverrides *imageoverride.Overrides } type windowsComponent struct { @@ -77,8 +82,8 @@ func (c *windowsComponent) ResolveImages(is *operatorv1.ImageSet) error { return imageName } - cniImage := imageoverride.Resolve(ComponentNameWindowsCNIImg, components.ComponentCalicoCNIWindows, c.cfg.Installation) - nodeImage := imageoverride.Resolve(ComponentNameWindowsNodeImg, components.ComponentCalicoNodeWindows, c.cfg.Installation) + cniImage := c.cfg.ImageOverrides.Resolve(ComponentNameWindowsCNIImg, components.ComponentCalicoCNIWindows, c.cfg.Installation) + nodeImage := c.cfg.ImageOverrides.Resolve(ComponentNameWindowsNodeImg, components.ComponentCalicoNodeWindows, c.cfg.Installation) c.cniImage = appendIfErr(components.GetReference(cniImage, reg, path, prefix, is)) c.nodeImage = appendIfErr(components.GetReference(nodeImage, reg, path, prefix, is)) diff --git a/pkg/render/windows_test.go b/pkg/render/windows_test.go index 5e845b31d5..e92bc86478 100644 --- a/pkg/render/windows_test.go +++ b/pkg/render/windows_test.go @@ -54,7 +54,7 @@ func renderWindows(cfg *render.WindowsConfiguration) []client.Object { if p, ok := comp.(render.ExtensionContextProvider); ok { rc.Component = p.ExtensionContext() } - out, _ := extensions.ApplyModifiers(render.ComponentNameWindows, rc, objs, nil) + out, _ := ext.ApplyModifiers(render.ComponentNameWindows, rc, objs, nil) return out } @@ -124,12 +124,13 @@ var _ = Describe("Windows rendering tests", func() { // Create a default configuration. cfg = render.WindowsConfiguration{ - K8sServiceEp: k8sServiceEp, - K8sDNSServers: []string{"10.96.0.10"}, - Installation: defaultInstance, - ClusterDomain: defaultClusterDomain, - TLS: typhaNodeTLS, - VXLANVNI: 4096, + K8sServiceEp: k8sServiceEp, + K8sDNSServers: []string{"10.96.0.10"}, + Installation: defaultInstance, + ClusterDomain: defaultClusterDomain, + TLS: typhaNodeTLS, + VXLANVNI: 4096, + ImageOverrides: ext.Images(), } }) From cc48b455e7109407d9053211f0f1e73d47f4c8a8 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Wed, 17 Jun 2026 13:45:54 -0700 Subject: [PATCH 27/38] Use r.opts for controller options instead of copied fields The installation, windows, and clusterconnection reconcilers copied a handful of ControllerOptions values into their own struct fields. Drop those and read them off r.opts so the options live in one place. Also removes the dead kubernetesVersion field on the installation reconciler. --- .../clusterconnection_controller.go | 6 +- .../installation/core_controller.go | 89 ++-- .../installation/core_controller_test.go | 400 ++++++++++-------- .../installation/windows_controller.go | 18 +- .../installation/windows_controller_test.go | 22 +- 5 files changed, 277 insertions(+), 258 deletions(-) diff --git a/pkg/controller/clusterconnection/clusterconnection_controller.go b/pkg/controller/clusterconnection/clusterconnection_controller.go index 763743789f..c51bc19238 100644 --- a/pkg/controller/clusterconnection/clusterconnection_controller.go +++ b/pkg/controller/clusterconnection/clusterconnection_controller.go @@ -174,7 +174,6 @@ func newReconciler( scheme: schema, provider: p, status: statusMgr, - clusterDomain: opts.ClusterDomain, tierWatchReady: tierWatchReady, clusterInfoWatchReady: clusterInfoWatchReady, opts: opts, @@ -192,7 +191,6 @@ type ReconcileConnection struct { scheme *runtime.Scheme provider operatorv1.Provider status status.StatusManager - clusterDomain string tierWatchReady *utils.ReadyFlag clusterInfoWatchReady *utils.ReadyFlag resolvedPodProxies []*httpproxy.Config @@ -286,7 +284,7 @@ func (r *ReconcileConnection) Reconcile(ctx context.Context, request reconcile.R log.V(2).Info("Loaded ManagementClusterConnection config", "config", managementClusterConnection) - certificateManager, err := certificatemanager.Create(r.cli, installationSpec, r.clusterDomain, common.OperatorNamespace(), certificatemanager.WithLogger(reqLogger)) + certificateManager, err := certificatemanager.Create(r.cli, installationSpec, r.opts.ClusterDomain, common.OperatorNamespace(), certificatemanager.WithLogger(reqLogger)) if err != nil { r.status.SetDegraded(operatorv1.ResourceCreateError, "Unable to create the Tigera CA", err, reqLogger) return reconcile.Result{}, err @@ -310,7 +308,7 @@ func (r *ReconcileConnection) Reconcile(ctx context.Context, request reconcile.R var guardianKeyPair certificatemanagement.KeyPairInterface if !variant.IsEnterprise() { - guardianCertificateNames := dns.GetServiceDNSNames("guardian", render.GuardianNamespace, r.clusterDomain) + guardianCertificateNames := dns.GetServiceDNSNames("guardian", render.GuardianNamespace, r.opts.ClusterDomain) guardianCertificateNames = append(guardianCertificateNames, "localhost", "127.0.0.1") guardianKeyPair, err = certificateManager.GetOrCreateKeyPair(r.cli, render.GuardianKeyPairSecret, whisker.WhiskerNamespace, guardianCertificateNames) if err != nil { diff --git a/pkg/controller/installation/core_controller.go b/pkg/controller/installation/core_controller.go index 1f241da0a8..08444f08e8 100644 --- a/pkg/controller/installation/core_controller.go +++ b/pkg/controller/installation/core_controller.go @@ -43,7 +43,6 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/cache" @@ -336,26 +335,17 @@ func newReconciler(mgr manager.Manager, opts options.ControllerOptions) (*Reconc typhaScaler := newTyphaAutoscaler(opts.K8sClientset, nodeIndexInformer, typhaListWatch, statusManager) r := &ReconcileInstallation{ - config: mgr.GetConfig(), - client: mgr.GetClient(), - clientset: opts.K8sClientset, - scheme: mgr.GetScheme(), - shutdownContext: opts.ShutdownContext, - watches: make(map[runtime.Object]struct{}), - autoDetectedProvider: opts.DetectedProvider, - status: statusManager, - typhaAutoscaler: typhaScaler, - namespaceMigration: nm, - enterpriseCRDsExist: opts.EnterpriseCRDExists, - clusterDomain: opts.ClusterDomain, - manageCRDs: opts.ManageCRDs, - tierWatchReady: &utils.ReadyFlag{}, - migrationWatchReady: &utils.ReadyFlag{}, - newComponentHandler: utils.NewComponentHandler, - v3CRDs: opts.UseV3CRDs, - kubernetesVersion: opts.KubernetesVersion, - apiDiscovery: opts.APIDiscovery, - opts: opts, + config: mgr.GetConfig(), + client: mgr.GetClient(), + scheme: mgr.GetScheme(), + watches: make(map[runtime.Object]struct{}), + status: statusManager, + typhaAutoscaler: typhaScaler, + namespaceMigration: nm, + tierWatchReady: &utils.ReadyFlag{}, + migrationWatchReady: &utils.ReadyFlag{}, + newComponentHandler: utils.NewComponentHandler, + opts: opts, } r.status.Run(opts.ShutdownContext) r.typhaAutoscaler.start(opts.ShutdownContext) @@ -397,24 +387,15 @@ type ReconcileInstallation struct { // that reads objects from the cache and writes to the apiserver config *rest.Config client client.Client - clientset *kubernetes.Clientset scheme *runtime.Scheme - shutdownContext context.Context watches map[runtime.Object]struct{} - autoDetectedProvider operatorv1.Provider status status.StatusManager typhaAutoscaler *typhaAutoscaler typhaAutoscalerNonClusterHost *typhaAutoscaler namespaceMigration migration.NamespaceMigration - enterpriseCRDsExist bool migrationChecked bool - clusterDomain string - manageCRDs bool tierWatchReady *utils.ReadyFlag migrationWatchReady *utils.ReadyFlag - v3CRDs bool - kubernetesVersion *common.VersionInfo - apiDiscovery *discovery.APIDiscovery opts options.ControllerOptions // newComponentHandler returns a new component handler. Useful stub for unit testing. @@ -859,7 +840,7 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile } // update Installation with defaults - if err := updateInstallationWithDefaults(ctx, r.client, instance, r.autoDetectedProvider); err != nil { + if err := updateInstallationWithDefaults(ctx, r.client, instance, r.opts.DetectedProvider); err != nil { r.status.SetDegraded(operatorv1.ResourceReadError, "Error querying installation", err, reqLogger) return reconcile.Result{}, err } @@ -1023,10 +1004,10 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile // The operator supports running in a "Calico only" mode so that it doesn't need to run enterprise-specific controllers. // If we are switching from this mode to one that enables enterprise, we need to restart the operator to enable the other controllers. - if !r.enterpriseCRDsExist && instance.Spec.Variant.IsEnterprise() { + if !r.opts.EnterpriseCRDExists && instance.Spec.Variant.IsEnterprise() { // Perform an API discovery to determine if the necessary APIs exist. If they do, we can reboot into enterprise mode. // if they do not, we need to notify the user that the requested configuration is invalid. - b, err := discovery.RequiresTigeraSecure(r.clientset) + b, err := discovery.RequiresTigeraSecure(r.opts.K8sClientset) if b { log.Info("Rebooting to enable TigeraSecure controllers") os.Exit(0) @@ -1052,7 +1033,7 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile var managementCluster *operatorv1.ManagementCluster var managementClusterConnection *operatorv1.ManagementClusterConnection var logCollector *operatorv1.LogCollector - if r.enterpriseCRDsExist { + if r.opts.EnterpriseCRDExists { logCollector, err = utils.GetLogCollector(ctx, r.client) if logCollector != nil { if err != nil { @@ -1098,7 +1079,7 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile } } - certificateManager, err := certificatemanager.Create(r.client, &instance.Spec, r.clusterDomain, common.OperatorNamespace(), certificatemanager.WithLogger(reqLogger)) + certificateManager, err := certificatemanager.Create(r.client, &instance.Spec, r.opts.ClusterDomain, common.OperatorNamespace(), certificatemanager.WithLogger(reqLogger)) if err != nil { r.status.SetDegraded(operatorv1.ResourceCreateError, "Unable to create the Tigera CA", err, reqLogger) return reconcile.Result{}, err @@ -1235,7 +1216,7 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile FelixConfiguration: felixConfiguration, CertificateManager: certificateManager, TrustedBundle: typhaNodeTLS.TrustedBundle, - ClusterDomain: r.clusterDomain, + ClusterDomain: r.opts.ClusterDomain, }) if err != nil { r.status.SetDegraded(operatorv1.ResourceCreateError, "Error preparing installation extension", err, reqLogger) @@ -1256,7 +1237,7 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile r.client, kubecontrollers.KubeControllerPrometheusTLSSecret, common.OperatorNamespace(), - dns.GetServiceDNSNames(kubecontrollers.KubeControllerMetrics, common.CalicoNamespace, r.clusterDomain)) + dns.GetServiceDNSNames(kubecontrollers.KubeControllerMetrics, common.CalicoNamespace, r.opts.ClusterDomain)) if err != nil { r.status.SetDegraded(operatorv1.ResourceReadError, "Error finding or creating TLS certificate kube controllers metric", err, reqLogger) return reconcile.Result{}, err @@ -1370,7 +1351,7 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile r.client, applicationlayer.WAFWebhookServerTLSSecretName, common.OperatorNamespace(), - dns.GetServiceDNSNames(applicationlayer.WAFWebhookServiceName, common.CalicoNamespace, r.clusterDomain)) + dns.GetServiceDNSNames(applicationlayer.WAFWebhookServiceName, common.CalicoNamespace, r.opts.ClusterDomain)) if err != nil { r.status.SetDegraded(operatorv1.ResourceCreateError, "Error creating WAF admission webhook TLS certificate", err, reqLogger) return reconcile.Result{}, err @@ -1429,11 +1410,11 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile hepListWatch := cache.NewListWatchFromClient(calicoClient.ProjectcalicoV3().RESTClient(), "hostendpoints", corev1.NamespaceAll, fields.Everything()) hepIndexInformer := cache.NewSharedIndexInformer(hepListWatch, &v3.HostEndpoint{}, 0, cache.Indexers{}) - go hepIndexInformer.Run(r.shutdownContext.Done()) + go hepIndexInformer.Run(r.opts.ShutdownContext.Done()) - typhaNonClusterHostWatch := cache.NewListWatchFromClient(r.clientset.AppsV1().RESTClient(), "deployments", "calico-system", fields.OneTermEqualSelector("metadata.name", "calico-typha"+render.TyphaNonClusterHostSuffix)) - r.typhaAutoscalerNonClusterHost = newTyphaAutoscaler(r.clientset, hepIndexInformer, typhaNonClusterHostWatch, r.status, typhaAutoscalerOptionNonclusterHost(true)) - r.typhaAutoscalerNonClusterHost.start(r.shutdownContext) + typhaNonClusterHostWatch := cache.NewListWatchFromClient(r.opts.K8sClientset.AppsV1().RESTClient(), "deployments", "calico-system", fields.OneTermEqualSelector("metadata.name", "calico-typha"+render.TyphaNonClusterHostSuffix)) + r.typhaAutoscalerNonClusterHost = newTyphaAutoscaler(r.opts.K8sClientset, hepIndexInformer, typhaNonClusterHostWatch, r.status, typhaAutoscalerOptionNonclusterHost(true)) + r.typhaAutoscalerNonClusterHost.start(r.opts.ShutdownContext) } } } @@ -1445,7 +1426,7 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile Installation: &instance.Spec, TLS: typhaNodeTLS, MigrateNamespaces: needsNamespaceMigration, - ClusterDomain: r.clusterDomain, + ClusterDomain: r.opts.ClusterDomain, NonClusterHost: nonclusterhost, FelixHealthPort: *felixConfiguration.Spec.HealthPort, } @@ -1568,7 +1549,7 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile LogCollector: logCollector, BirdTemplates: birdTemplates, TLS: typhaNodeTLS, - ClusterDomain: r.clusterDomain, + ClusterDomain: r.opts.ClusterDomain, DefaultDNSPolicy: defaultDNSPolicy, DefaultDNSConfig: defaultDNSConfig, GoldmaneIP: goldmaneIP, @@ -1578,7 +1559,7 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile CanRemoveCNIFinalizer: canRemoveCNI, FelixHealthPort: *felixConfiguration.Spec.HealthPort, NodeCgroupV2Path: felixConfiguration.Spec.CgroupV2Path, - V3CRDs: r.v3CRDs, + V3CRDs: r.opts.UseV3CRDs, ImageOverrides: r.opts.Extensions.Images(), } @@ -1656,7 +1637,7 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile Installation: &instance.Spec, ManagementCluster: managementCluster, ManagementClusterConnection: managementClusterConnection, - ClusterDomain: r.clusterDomain, + ClusterDomain: r.opts.ClusterDomain, MetricsPort: kubeControllersMetricsPort, Terminating: installationMarkedForDeletion, MetricsServerTLS: kubeControllerTLS, @@ -2314,10 +2295,10 @@ func (r *ReconcileInstallation) checkActive(log logr.Logger) (*corev1.ConfigMap, } func (r *ReconcileInstallation) updateCRDs(ctx context.Context, variant operatorv1.ProductVariant, log logr.Logger) error { - if !r.manageCRDs { + if !r.opts.ManageCRDs { return nil } - crdComponent := render.NewCreationPassthrough(crds.ToRuntimeObjects(crds.GetCRDs(variant, r.v3CRDs)...)...) + crdComponent := render.NewCreationPassthrough(crds.ToRuntimeObjects(crds.GetCRDs(variant, r.opts.UseV3CRDs)...)...) // Specify nil for the CR so no ownership is put on the CRDs. We do this so removing the // Installation CR will not remove the CRDs. handler := r.newComponentHandler(log, r.client, r.scheme, nil) @@ -2329,19 +2310,19 @@ func (r *ReconcileInstallation) updateCRDs(ctx context.Context, variant operator } func (r *ReconcileInstallation) updateMutatingAdmissionPolicies(ctx context.Context, install *operatorv1.Installation, log logr.Logger) error { - if !r.manageCRDs || !r.v3CRDs { + if !r.opts.ManageCRDs || !r.opts.UseV3CRDs { return nil } // MutatingAdmissionPolicy served version was discovered once at startup (v1 was promoted to GA // in k8s 1.36 and v1beta1 (introduced in 1.32) is scheduled for removal in 1.37). - mapAPIVersion := r.apiDiscovery.ServedVersion(admission.APIGroup, admission.KindPolicy) + mapAPIVersion := r.opts.APIDiscovery.ServedVersion(admission.APIGroup, admission.KindPolicy) if mapAPIVersion == "" { r.status.SetDegraded(operatorv1.ResourceNotReady, "Kubernetes cluster does not serve MutatingAdmissionPolicy (requires v1.32+); policy defaulting will not be available", nil, log) return nil } - desired := admission.GetMutatingAdmissionPolicies(install.Spec.Variant, r.v3CRDs, mapAPIVersion) + desired := admission.GetMutatingAdmissionPolicies(install.Spec.Variant, r.opts.UseV3CRDs, mapAPIVersion) existingMAPs, existingMAPBs, err := admission.ListManaged(ctx, r.client, mapAPIVersion) if err != nil { r.status.SetDegraded(operatorv1.ResourceReadError, "Error listing managed MutatingAdmissionPolicy resources", err, log) @@ -2352,20 +2333,20 @@ func (r *ReconcileInstallation) updateMutatingAdmissionPolicies(ctx context.Cont } func (r *ReconcileInstallation) updateValidatingAdmissionPolicies(ctx context.Context, install *operatorv1.Installation, log logr.Logger) error { - if !r.manageCRDs || !r.v3CRDs { + if !r.opts.ManageCRDs || !r.opts.UseV3CRDs { return nil } // ValidatingAdmissionPolicy reached GA (v1) well before MutatingAdmissionPolicy, so it has its own // served version and is reconciled independently of whether the cluster serves MAPs. If the cluster // doesn't serve it at all there's nothing to do, so skip rather than degrade. - vapAPIVersion := r.apiDiscovery.ServedVersion(admission.APIGroup, admission.KindValidatingPolicy) + vapAPIVersion := r.opts.APIDiscovery.ServedVersion(admission.APIGroup, admission.KindValidatingPolicy) if vapAPIVersion == "" { log.Info("Kubernetes cluster does not serve ValidatingAdmissionPolicy, skipping") return nil } - desired := admission.GetValidatingAdmissionPolicies(install.Spec.Variant, r.v3CRDs, vapAPIVersion) + desired := admission.GetValidatingAdmissionPolicies(install.Spec.Variant, r.opts.UseV3CRDs, vapAPIVersion) existingVAPs, existingVAPBs, err := admission.ListManagedValidating(ctx, r.client, vapAPIVersion) if err != nil { r.status.SetDegraded(operatorv1.ResourceReadError, "Error listing managed ValidatingAdmissionPolicy resources", err, log) diff --git a/pkg/controller/installation/core_controller_test.go b/pkg/controller/installation/core_controller_test.go index d2e73fe0ad..3fcd559551 100644 --- a/pkg/controller/installation/core_controller_test.go +++ b/pkg/controller/installation/core_controller_test.go @@ -191,19 +191,21 @@ var _ = Describe("Testing core-controller installation", func() { // As the parameters in the client changes, we expect the outcomes of the reconcile loops to change. r = ReconcileInstallation{ - opts: options.ControllerOptions{Extensions: testExtensions}, - config: nil, // there is no fake for config - client: c, - scheme: scheme, - autoDetectedProvider: operator.ProviderNone, - status: mockStatus, - typhaAutoscaler: newTyphaAutoscaler(cs, nodeIndexInformer, test.NewTyphaListWatch(cs), mockStatus), - namespaceMigration: &fakeNamespaceMigration{}, - enterpriseCRDsExist: true, - migrationChecked: true, - tierWatchReady: ready, - migrationWatchReady: &utils.ReadyFlag{}, - newComponentHandler: utils.NewComponentHandler, + opts: options.ControllerOptions{ + Extensions: testExtensions, + DetectedProvider: operator.ProviderNone, + EnterpriseCRDExists: true, + }, + config: nil, // there is no fake for config + client: c, + scheme: scheme, + status: mockStatus, + typhaAutoscaler: newTyphaAutoscaler(cs, nodeIndexInformer, test.NewTyphaListWatch(cs), mockStatus), + namespaceMigration: &fakeNamespaceMigration{}, + migrationChecked: true, + tierWatchReady: ready, + migrationWatchReady: &utils.ReadyFlag{}, + newComponentHandler: utils.NewComponentHandler, } r.typhaAutoscaler.start(ctx) @@ -821,20 +823,22 @@ var _ = Describe("Testing core-controller installation", func() { // As the parameters in the client changes, we expect the outcomes of the reconcile loops to change. r = ReconcileInstallation{ - opts: options.ControllerOptions{Extensions: testExtensions}, - config: nil, // there is no fake for config - client: c, - scheme: scheme, - autoDetectedProvider: operator.ProviderNone, - status: mockStatus, - typhaAutoscaler: newTyphaAutoscaler(cs, nodeIndexInformer, test.NewTyphaListWatch(cs), mockStatus), - namespaceMigration: &fakeNamespaceMigration{}, - enterpriseCRDsExist: true, - migrationChecked: true, - clusterDomain: dns.DefaultClusterDomain, - tierWatchReady: ready, - migrationWatchReady: &utils.ReadyFlag{}, - newComponentHandler: utils.NewComponentHandler, + opts: options.ControllerOptions{ + Extensions: testExtensions, + DetectedProvider: operator.ProviderNone, + EnterpriseCRDExists: true, + ClusterDomain: dns.DefaultClusterDomain, + }, + config: nil, // there is no fake for config + client: c, + scheme: scheme, + status: mockStatus, + typhaAutoscaler: newTyphaAutoscaler(cs, nodeIndexInformer, test.NewTyphaListWatch(cs), mockStatus), + namespaceMigration: &fakeNamespaceMigration{}, + migrationChecked: true, + tierWatchReady: ready, + migrationWatchReady: &utils.ReadyFlag{}, + newComponentHandler: utils.NewComponentHandler, } r.typhaAutoscaler.start(ctx) @@ -1044,19 +1048,21 @@ var _ = Describe("Testing core-controller installation", func() { // As the parameters in the client changes, we expect the outcomes of the reconcile loops to change. r = ReconcileInstallation{ - opts: options.ControllerOptions{Extensions: testExtensions}, - config: nil, // there is no fake for config - client: c, - scheme: scheme, - autoDetectedProvider: operator.ProviderNone, - status: mockStatus, - typhaAutoscaler: newTyphaAutoscaler(cs, nodeIndexInformer, test.NewTyphaListWatch(cs), mockStatus), - namespaceMigration: &fakeNamespaceMigration{}, - enterpriseCRDsExist: true, - migrationChecked: true, - tierWatchReady: ready, - migrationWatchReady: &utils.ReadyFlag{}, - newComponentHandler: utils.NewComponentHandler, + opts: options.ControllerOptions{ + Extensions: testExtensions, + DetectedProvider: operator.ProviderNone, + EnterpriseCRDExists: true, + }, + config: nil, // there is no fake for config + client: c, + scheme: scheme, + status: mockStatus, + typhaAutoscaler: newTyphaAutoscaler(cs, nodeIndexInformer, test.NewTyphaListWatch(cs), mockStatus), + namespaceMigration: &fakeNamespaceMigration{}, + migrationChecked: true, + tierWatchReady: ready, + migrationWatchReady: &utils.ReadyFlag{}, + newComponentHandler: utils.NewComponentHandler, } r.typhaAutoscaler.start(ctx) @@ -2221,7 +2227,7 @@ var _ = Describe("Testing core-controller installation", func() { cr.Spec.Variant = operator.Calico cr.Status.Variant = operator.Calico Expect(c.Create(ctx, cr)).NotTo(HaveOccurred()) - r.enterpriseCRDsExist = false + r.opts.EnterpriseCRDExists = false Expect(c.Delete(ctx, &v3.Tier{ObjectMeta: metav1.ObjectMeta{Name: "calico-system"}})).NotTo(HaveOccurred()) _, err := r.Reconcile(ctx, reconcile.Request{}) @@ -2333,20 +2339,22 @@ var _ = Describe("Testing core-controller installation", func() { // As the parameters in the client changes, we expect the outcomes of the reconcile loops to change. r = ReconcileInstallation{ - opts: options.ControllerOptions{Extensions: testExtensions}, - config: nil, // there is no fake for config - client: c, - scheme: scheme, - autoDetectedProvider: operator.ProviderNone, - status: mockStatus, - typhaAutoscaler: newTyphaAutoscaler(cs, nodeIndexInformer, test.NewTyphaListWatch(cs), mockStatus), - namespaceMigration: &fakeNamespaceMigration{}, - enterpriseCRDsExist: true, - migrationChecked: true, - clusterDomain: dns.DefaultClusterDomain, - tierWatchReady: ready, - migrationWatchReady: &utils.ReadyFlag{}, - newComponentHandler: utils.NewComponentHandler, + opts: options.ControllerOptions{ + Extensions: testExtensions, + DetectedProvider: operator.ProviderNone, + EnterpriseCRDExists: true, + ClusterDomain: dns.DefaultClusterDomain, + }, + config: nil, // there is no fake for config + client: c, + scheme: scheme, + status: mockStatus, + typhaAutoscaler: newTyphaAutoscaler(cs, nodeIndexInformer, test.NewTyphaListWatch(cs), mockStatus), + namespaceMigration: &fakeNamespaceMigration{}, + migrationChecked: true, + tierWatchReady: ready, + migrationWatchReady: &utils.ReadyFlag{}, + newComponentHandler: utils.NewComponentHandler, } r.typhaAutoscaler.start(ctx) @@ -2471,18 +2479,20 @@ var _ = Describe("Testing core-controller installation", func() { componentHandler = newFakeComponentHandler() r = ReconcileInstallation{ - opts: options.ControllerOptions{Extensions: testExtensions}, - config: nil, // there is no fake for config - client: c, - scheme: scheme, - autoDetectedProvider: operator.ProviderNone, - status: mockStatus, - typhaAutoscaler: newTyphaAutoscaler(cs, nodeIndexInformer, test.NewTyphaListWatch(cs), mockStatus), - namespaceMigration: &fakeNamespaceMigration{}, - enterpriseCRDsExist: true, - migrationChecked: true, - tierWatchReady: ready, - migrationWatchReady: &utils.ReadyFlag{}, + opts: options.ControllerOptions{ + Extensions: testExtensions, + DetectedProvider: operator.ProviderNone, + EnterpriseCRDExists: true, + }, + config: nil, // there is no fake for config + client: c, + scheme: scheme, + status: mockStatus, + typhaAutoscaler: newTyphaAutoscaler(cs, nodeIndexInformer, test.NewTyphaListWatch(cs), mockStatus), + namespaceMigration: &fakeNamespaceMigration{}, + migrationChecked: true, + tierWatchReady: ready, + migrationWatchReady: &utils.ReadyFlag{}, newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, @@ -2635,13 +2645,15 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { It("should create v1 MAPs when v1 is served", func() { r = ReconcileInstallation{ - opts: options.ControllerOptions{Extensions: testExtensions}, - client: clientFor(), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: true, - apiDiscovery: discoveryFor(admission.VersionV1), + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: true, + APIDiscovery: discoveryFor(admission.VersionV1), + }, + client: clientFor(), + scheme: scheme, + status: mockStatus, newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, @@ -2667,13 +2679,15 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { It("should create v1beta1 MAPs when only v1beta1 is served", func() { r = ReconcileInstallation{ - opts: options.ControllerOptions{Extensions: testExtensions}, - client: clientFor(), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: true, - apiDiscovery: discoveryFor(admission.VersionV1Beta1), + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: true, + APIDiscovery: discoveryFor(admission.VersionV1Beta1), + }, + client: clientFor(), + scheme: scheme, + status: mockStatus, newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, @@ -2697,13 +2711,15 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { It("should create v1alpha1 MAPs when only v1alpha1 is served", func() { r = ReconcileInstallation{ - opts: options.ControllerOptions{Extensions: testExtensions}, - client: clientFor(), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: true, - apiDiscovery: discoveryFor(admission.VersionV1Alpha1), + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: true, + APIDiscovery: discoveryFor(admission.VersionV1Alpha1), + }, + client: clientFor(), + scheme: scheme, + status: mockStatus, newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, @@ -2727,13 +2743,15 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { It("should not create MAPs when no served version exists and should set degraded", func() { r = ReconcileInstallation{ - opts: options.ControllerOptions{Extensions: testExtensions}, - client: clientFor(), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: true, - apiDiscovery: discoveryFor(""), + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: true, + APIDiscovery: discoveryFor(""), + }, + client: clientFor(), + scheme: scheme, + status: mockStatus, newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, @@ -2746,13 +2764,15 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { It("should not create MAPs when v3CRDs=false", func() { r = ReconcileInstallation{ - opts: options.ControllerOptions{Extensions: testExtensions}, - client: clientFor(), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: false, - apiDiscovery: discoveryFor(admission.VersionV1), + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: false, + APIDiscovery: discoveryFor(admission.VersionV1), + }, + client: clientFor(), + scheme: scheme, + status: mockStatus, newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, @@ -2764,13 +2784,15 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { It("should not create MAPs when manageCRDs=false", func() { r = ReconcileInstallation{ - opts: options.ControllerOptions{Extensions: testExtensions}, - client: clientFor(), - scheme: scheme, - status: mockStatus, - manageCRDs: false, - v3CRDs: true, - apiDiscovery: discoveryFor(admission.VersionV1), + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: false, + UseV3CRDs: true, + APIDiscovery: discoveryFor(admission.VersionV1), + }, + client: clientFor(), + scheme: scheme, + status: mockStatus, newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, @@ -2795,13 +2817,15 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { } r = ReconcileInstallation{ - opts: options.ControllerOptions{Extensions: testExtensions}, - client: clientFor(staleMAP, staleMAPB), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: true, - apiDiscovery: discoveryFor(admission.VersionV1), + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: true, + APIDiscovery: discoveryFor(admission.VersionV1), + }, + client: clientFor(staleMAP, staleMAPB), + scheme: scheme, + status: mockStatus, newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, @@ -2838,13 +2862,15 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { } r = ReconcileInstallation{ - opts: options.ControllerOptions{Extensions: testExtensions}, - client: clientFor(initial...), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: true, - apiDiscovery: discoveryFor(admission.VersionV1), + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: true, + APIDiscovery: discoveryFor(admission.VersionV1), + }, + client: clientFor(initial...), + scheme: scheme, + status: mockStatus, newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, @@ -2857,13 +2883,15 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { It("should work with Enterprise variant", func() { r = ReconcileInstallation{ - opts: options.ControllerOptions{Extensions: testExtensions}, - client: clientFor(), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: true, - apiDiscovery: discoveryFor(admission.VersionV1), + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: true, + APIDiscovery: discoveryFor(admission.VersionV1), + }, + client: clientFor(), + scheme: scheme, + status: mockStatus, newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, @@ -2929,13 +2957,15 @@ var _ = Describe("updateValidatingAdmissionPolicies", func() { It("should create v1 VAPs when v1 is served", func() { r = ReconcileInstallation{ - opts: options.ControllerOptions{Extensions: testExtensions}, - client: clientFor(), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: true, - apiDiscovery: discoveryFor(admission.VersionV1), + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: true, + APIDiscovery: discoveryFor(admission.VersionV1), + }, + client: clientFor(), + scheme: scheme, + status: mockStatus, newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, @@ -2961,13 +2991,15 @@ var _ = Describe("updateValidatingAdmissionPolicies", func() { It("should create v1beta1 VAPs when only v1beta1 is served", func() { r = ReconcileInstallation{ - opts: options.ControllerOptions{Extensions: testExtensions}, - client: clientFor(), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: true, - apiDiscovery: discoveryFor(admission.VersionV1Beta1), + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: true, + APIDiscovery: discoveryFor(admission.VersionV1Beta1), + }, + client: clientFor(), + scheme: scheme, + status: mockStatus, newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, @@ -2991,13 +3023,15 @@ var _ = Describe("updateValidatingAdmissionPolicies", func() { It("should create v1alpha1 VAPs when only v1alpha1 is served", func() { r = ReconcileInstallation{ - opts: options.ControllerOptions{Extensions: testExtensions}, - client: clientFor(), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: true, - apiDiscovery: discoveryFor(admission.VersionV1Alpha1), + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: true, + APIDiscovery: discoveryFor(admission.VersionV1Alpha1), + }, + client: clientFor(), + scheme: scheme, + status: mockStatus, newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, @@ -3009,13 +3043,15 @@ var _ = Describe("updateValidatingAdmissionPolicies", func() { It("should skip without degrading when no served version exists", func() { r = ReconcileInstallation{ - opts: options.ControllerOptions{Extensions: testExtensions}, - client: clientFor(), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: true, - apiDiscovery: discoveryFor(""), + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: true, + APIDiscovery: discoveryFor(""), + }, + client: clientFor(), + scheme: scheme, + status: mockStatus, newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, @@ -3028,13 +3064,15 @@ var _ = Describe("updateValidatingAdmissionPolicies", func() { It("should not create VAPs when v3CRDs=false", func() { r = ReconcileInstallation{ - opts: options.ControllerOptions{Extensions: testExtensions}, - client: clientFor(), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: false, - apiDiscovery: discoveryFor(admission.VersionV1), + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: false, + APIDiscovery: discoveryFor(admission.VersionV1), + }, + client: clientFor(), + scheme: scheme, + status: mockStatus, newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, @@ -3059,13 +3097,15 @@ var _ = Describe("updateValidatingAdmissionPolicies", func() { } r = ReconcileInstallation{ - opts: options.ControllerOptions{Extensions: testExtensions}, - client: clientFor(staleVAP, staleVAPB), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: true, - apiDiscovery: discoveryFor(admission.VersionV1), + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: true, + APIDiscovery: discoveryFor(admission.VersionV1), + }, + client: clientFor(staleVAP, staleVAPB), + scheme: scheme, + status: mockStatus, newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, @@ -3084,13 +3124,15 @@ var _ = Describe("updateValidatingAdmissionPolicies", func() { It("should work with Enterprise variant", func() { r = ReconcileInstallation{ - opts: options.ControllerOptions{Extensions: testExtensions}, - client: clientFor(), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: true, - apiDiscovery: discoveryFor(admission.VersionV1), + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: true, + APIDiscovery: discoveryFor(admission.VersionV1), + }, + client: clientFor(), + scheme: scheme, + status: mockStatus, newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, diff --git a/pkg/controller/installation/windows_controller.go b/pkg/controller/installation/windows_controller.go index 12e31b2a1f..ad758dbbce 100644 --- a/pkg/controller/installation/windows_controller.go +++ b/pkg/controller/installation/windows_controller.go @@ -83,7 +83,7 @@ func AddWindowsController(mgr manager.Manager, opts options.ControllerOptions) e return fmt.Errorf("tigera-windows-controller failed to watch calico Tigerastatus: %w", err) } - if ri.autoDetectedProvider.IsOpenShift() { + if ri.opts.DetectedProvider.IsOpenShift() { // Watch for openshift network configuration as well. If we're running in OpenShift, we need to // merge this configuration with our own and the write back the status object. err = c.WatchObject(&configv1.Network{}, &handler.EnqueueRequestForObject{}) @@ -151,7 +151,7 @@ func AddWindowsController(mgr manager.Manager, opts options.ControllerOptions) e // Watch for changes to IPAMConfiguration. go utils.WaitToAddResourceWatch(c, opts.K8sClientset, logw, ri.ipamConfigWatchReady, []client.Object{&v3.IPAMConfiguration{TypeMeta: metav1.TypeMeta{Kind: v3.KindIPAMConfiguration}}}) - if ri.enterpriseCRDsExist { + if ri.opts.EnterpriseCRDExists { for _, ns := range []string{common.CalicoNamespace, common.OperatorNamespace()} { if err = utils.AddSecretsWatch(c, render.NodePrometheusTLSServerSecret, ns); err != nil { return fmt.Errorf("tigera-windows-controller failed to watch secret '%s' in '%s' namespace: %w", render.NodePrometheusTLSServerSecret, ns, err) @@ -177,10 +177,7 @@ type ReconcileWindows struct { client client.Client scheme *runtime.Scheme watches map[runtime.Object]struct{} - autoDetectedProvider operatorv1.Provider status status.StatusManager - enterpriseCRDsExist bool - clusterDomain string ipamConfigWatchReady *utils.ReadyFlag opts options.ControllerOptions } @@ -194,10 +191,7 @@ func newWindowsReconciler(mgr manager.Manager, opts options.ControllerOptions) ( client: mgr.GetClient(), scheme: mgr.GetScheme(), watches: make(map[runtime.Object]struct{}), - autoDetectedProvider: opts.DetectedProvider, status: statusManager, - enterpriseCRDsExist: opts.EnterpriseCRDExists, - clusterDomain: opts.ClusterDomain, ipamConfigWatchReady: &utils.ReadyFlag{}, opts: opts, } @@ -291,7 +285,7 @@ func (r *ReconcileWindows) Reconcile(ctx context.Context, request reconcile.Requ return reconcile.Result{}, err } - certificateManager, err := certificatemanager.Create(r.client, &instance.Spec, r.clusterDomain, common.OperatorNamespace()) + certificateManager, err := certificatemanager.Create(r.client, &instance.Spec, r.opts.ClusterDomain, common.OperatorNamespace()) if err != nil { r.status.SetDegraded(operatorv1.ResourceCreateError, "Unable to create the Tigera CA", err, reqLogger) return reconcile.Result{}, err @@ -351,7 +345,7 @@ func (r *ReconcileWindows) Reconcile(ctx context.Context, request reconcile.Requ } // The key pair is created by the core controller, so if it isn't set, requeue to wait until it is - nodePrometheusTLS, err = certificateManager.GetKeyPair(r.client, render.NodePrometheusTLSServerSecret, common.OperatorNamespace(), dns.GetServiceDNSNames(render.WindowsNodeMetricsService, common.CalicoNamespace, r.clusterDomain)) + nodePrometheusTLS, err = certificateManager.GetKeyPair(r.client, render.NodePrometheusTLSServerSecret, common.OperatorNamespace(), dns.GetServiceDNSNames(render.WindowsNodeMetricsService, common.CalicoNamespace, r.opts.ClusterDomain)) if err != nil { r.status.SetDegraded(operatorv1.ResourceCreateError, "Error getting TLS certificate", err, reqLogger) return reconcile.Result{}, err @@ -360,7 +354,7 @@ func (r *ReconcileWindows) Reconcile(ctx context.Context, request reconcile.Requ var component render.Component - kubeDNSServiceName := utils.GetDNSServiceName(r.autoDetectedProvider) + kubeDNSServiceName := utils.GetDNSServiceName(r.opts.DetectedProvider) kubeDNSService := &corev1.Service{} err = r.client.Get(ctx, kubeDNSServiceName, kubeDNSService) if err != nil { @@ -384,7 +378,7 @@ func (r *ReconcileWindows) Reconcile(ctx context.Context, request reconcile.Requ K8sServiceEp: k8sapi.Endpoint, K8sDNSServers: kubeDNSIPs, Installation: &instance.Spec, - ClusterDomain: r.clusterDomain, + ClusterDomain: r.opts.ClusterDomain, TLS: typhaNodeTLS, PrometheusServerTLS: nodePrometheusTLS, NodeReporterMetricsPort: nodeReporterMetricsPort, diff --git a/pkg/controller/installation/windows_controller_test.go b/pkg/controller/installation/windows_controller_test.go index 98121210e7..6fa1d2cf0d 100644 --- a/pkg/controller/installation/windows_controller_test.go +++ b/pkg/controller/installation/windows_controller_test.go @@ -120,13 +120,15 @@ var _ = Describe("windows-controller installation tests", func() { // As the parameters in the client changes, we expect the outcomes of the reconcile loops to change. r = ReconcileWindows{ - opts: options.ControllerOptions{Extensions: testExtensions}, + opts: options.ControllerOptions{ + Extensions: testExtensions, + DetectedProvider: operator.ProviderNone, + EnterpriseCRDExists: true, + }, config: nil, // there is no fake for config client: c, scheme: scheme, - autoDetectedProvider: operator.ProviderNone, status: mockStatus, - enterpriseCRDsExist: true, ipamConfigWatchReady: &utils.ReadyFlag{}, } r.ipamConfigWatchReady.MarkAsReady() @@ -157,7 +159,7 @@ var _ = Describe("windows-controller installation tests", func() { }, }, } - Expect(updateInstallationWithDefaults(ctx, r.client, cr, r.autoDetectedProvider)).NotTo(HaveOccurred()) + Expect(updateInstallationWithDefaults(ctx, r.client, cr, r.opts.DetectedProvider)).NotTo(HaveOccurred()) certificateManager, err := certificatemanager.Create(c, nil, "", common.OperatorNamespace(), certificatemanager.AllowCACreation()) Expect(err).NotTo(HaveOccurred()) prometheusTLS, err := certificateManager.GetOrCreateKeyPair(c, monitor.PrometheusClientTLSSecretName, common.OperatorNamespace(), []string{monitor.PrometheusClientTLSSecretName}) @@ -196,7 +198,7 @@ var _ = Describe("windows-controller installation tests", func() { cr.Status = operator.InstallationStatus{ Variant: operator.Calico, } - Expect(updateInstallationWithDefaults(ctx, r.client, cr, r.autoDetectedProvider)).NotTo(HaveOccurred()) + Expect(updateInstallationWithDefaults(ctx, r.client, cr, r.opts.DetectedProvider)).NotTo(HaveOccurred()) // Set serviceCIDRs in the installation (required for Calico for Windows) cr.Spec.ServiceCIDRs = []string{"10.96.0.0/12"} @@ -611,13 +613,15 @@ var _ = Describe("windows-controller installation tests", func() { // As the parameters in the client changes, we expect the outcomes of the reconcile loops to change. r = ReconcileWindows{ - opts: options.ControllerOptions{Extensions: testExtensions}, + opts: options.ControllerOptions{ + Extensions: testExtensions, + DetectedProvider: operator.ProviderNone, + EnterpriseCRDExists: true, + }, config: nil, // there is no fake for config client: c, scheme: scheme, - autoDetectedProvider: operator.ProviderNone, status: mockStatus, - enterpriseCRDsExist: true, ipamConfigWatchReady: &utils.ReadyFlag{}, } r.ipamConfigWatchReady.MarkAsReady() @@ -666,7 +670,7 @@ var _ = Describe("windows-controller installation tests", func() { }, }, } - Expect(updateInstallationWithDefaults(ctx, r.client, instance, r.autoDetectedProvider)).NotTo(HaveOccurred()) + Expect(updateInstallationWithDefaults(ctx, r.client, instance, r.opts.DetectedProvider)).NotTo(HaveOccurred()) Expect(c.Create(ctx, instance)).NotTo(HaveOccurred()) }) AfterEach(func() { From 8b5349e365f984a4764f46e6d4a6f0dd64d5a0c0 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Thu, 18 Jun 2026 09:00:37 -0700 Subject: [PATCH 28/38] Apply variant extensions by decorating the rendered component Set.Decorate wraps a component so its objects pass through the registered modifier, and the wrapped value is itself a render.Component the handler renders like any other. Rename Extension to ComponentExtension. --- pkg/controller/utils/component.go | 26 +++++----- pkg/controller/utils/component_test.go | 2 +- pkg/enterprise/apiserver.go | 4 +- pkg/enterprise/decorate_helpers_test.go | 65 +++++++++++++++++++++++++ pkg/enterprise/guardian.go | 4 +- pkg/enterprise/guardian_test.go | 10 ++-- pkg/enterprise/node.go | 4 +- pkg/enterprise/node_test.go | 20 ++++---- pkg/enterprise/typha.go | 2 +- pkg/enterprise/typha_test.go | 6 +-- pkg/enterprise/windows.go | 6 +-- pkg/enterprise/windows_test.go | 10 ++-- pkg/extensions/decorate_helpers_test.go | 65 +++++++++++++++++++++++++ pkg/extensions/extension.go | 55 +++++++++++++++------ pkg/extensions/extension_test.go | 22 ++++----- pkg/extensions/image_test.go | 2 +- pkg/extensions/set.go | 4 +- pkg/render/apiserver_test.go | 2 +- pkg/render/decorate_helpers_test.go | 65 +++++++++++++++++++++++++ pkg/render/guardian_test.go | 6 +-- pkg/render/node_enterprise_test.go | 4 +- pkg/render/windows_test.go | 2 +- 22 files changed, 303 insertions(+), 83 deletions(-) create mode 100644 pkg/enterprise/decorate_helpers_test.go create mode 100644 pkg/extensions/decorate_helpers_test.go create mode 100644 pkg/render/decorate_helpers_test.go diff --git a/pkg/controller/utils/component.go b/pkg/controller/utils/component.go index 76db4ebf44..f805aef659 100644 --- a/pkg/controller/utils/component.go +++ b/pkg/controller/utils/component.go @@ -458,6 +458,18 @@ func resetMetadataForCreate(obj client.Object) { } func (c *componentHandler) CreateOrUpdateOrDelete(ctx context.Context, component render.Component, status status.StatusManager) error { + // Decorate the component with any registered variant extension before doing + // anything with it, so Ready, SupportedOSType, and Objects all reflect the + // extended component. Decorate is a no-op for components with no registered + // extension. + if ext, ok := component.(render.Extensible); ok && c.extensions == nil { + // The component can be extended but this handler was built without an + // extension Set, so any registered extension silently won't run. That is + // a wiring bug in the controller, not a normal state. + c.log.Info("BUG: extensible component rendered by a handler with no extension Set; extensions will not be applied", "component", ext.ModifierKey()) + } + component = c.extensions.Decorate(component, c.renderCtx) + // Before creating the component, make sure that it is ready. This provides a hook to do // dependency checking for the component. cmpLog := c.log.WithValues("component", reflect.TypeOf(component)) @@ -476,20 +488,6 @@ func (c *componentHandler) CreateOrUpdateOrDelete(ctx context.Context, component var cronJobs []types.NamespacedName objsToCreate, objsToDelete := component.Objects() - if ext, ok := component.(render.Extensible); ok { - if c.extensions == nil { - // The component can be extended but this handler was built without an - // extension Set, so any registered modifier silently won't run. That is - // a wiring bug in the controller, not a normal state. - c.log.Info("BUG: extensible component rendered by a handler with no extension Set; modifiers will not be applied", "component", ext.ModifierKey()) - } else { - rc := c.renderCtx - if p, ok := component.(render.ExtensionContextProvider); ok { - rc.Component = p.ExtensionContext() - } - objsToCreate, objsToDelete = c.extensions.ApplyModifiers(ext.ModifierKey(), rc, objsToCreate, objsToDelete) - } - } // Load the InstallationSpec once and reuse it for every object: createOrUpdateObject needs it // for image pull policy and TLS ciphers, and we use it here to decide whether the user has diff --git a/pkg/controller/utils/component_test.go b/pkg/controller/utils/component_test.go index d8c227d47b..ec369e2ac3 100644 --- a/pkg/controller/utils/component_test.go +++ b/pkg/controller/utils/component_test.go @@ -2580,7 +2580,7 @@ func (mc *mockClient) SubResource(subResource string) client.SubResourceClient { var _ = Describe("componentHandler modifier application", func() { It("applies registered modifiers to a named component before create", func() { ext := extensions.NewSet() - ext.Register(operatorv1.CalicoEnterprise, "fake", extensions.Extension{ + ext.Register(operatorv1.CalicoEnterprise, "fake", extensions.ComponentExtension{ Modify: func(ctx extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { cm := objs[0].(*corev1.ConfigMap) cm.Data = map[string]string{"patched": "yes"} diff --git a/pkg/enterprise/apiserver.go b/pkg/enterprise/apiserver.go index 1b7119f239..913391cb16 100644 --- a/pkg/enterprise/apiserver.go +++ b/pkg/enterprise/apiserver.go @@ -53,12 +53,12 @@ type apiServer struct { } func registerAPIServer(s *extensions.Set) { - s.Register(operatorv1.CalicoEnterprise, render.ComponentNameAPIServer, extensions.Extension{ + s.Register(operatorv1.CalicoEnterprise, render.ComponentNameAPIServer, extensions.ComponentExtension{ Modify: modifyAPIServer, }) // When running Calico, clean up any Enterprise objects left behind by a prior // Enterprise installation. - s.Register(operatorv1.Calico, render.ComponentNameAPIServer, extensions.Extension{ + s.Register(operatorv1.Calico, render.ComponentNameAPIServer, extensions.ComponentExtension{ Modify: cleanupAPIServer, }) } diff --git a/pkg/enterprise/decorate_helpers_test.go b/pkg/enterprise/decorate_helpers_test.go new file mode 100644 index 0000000000..5bade86c8f --- /dev/null +++ b/pkg/enterprise/decorate_helpers_test.go @@ -0,0 +1,65 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enterprise_test + +import ( + client "sigs.k8s.io/controller-runtime/pkg/client" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/extensions" + rmeta "github.com/tigera/operator/pkg/render/common/meta" +) + +// stubExtComponent adapts raw object lists to a render.Component so a registered +// extension can be exercised through Set.Decorate, the same seam the component +// handler uses. key selects the extension; extCtx is delivered as the per-component +// ExtensionContext. +type stubExtComponent struct { + key string + extCtx any + create, delete []client.Object +} + +func (s stubExtComponent) ResolveImages(*operatorv1.ImageSet) error { + return nil +} + +func (s stubExtComponent) Objects() ([]client.Object, []client.Object) { + return s.create, s.delete +} + +func (s stubExtComponent) Ready() bool { + return true +} + +func (s stubExtComponent) SupportedOSType() rmeta.OSType { + return rmeta.OSTypeAny +} + +func (s stubExtComponent) ModifierKey() string { + return s.key +} + +func (s stubExtComponent) ExtensionContext() any { + return s.extCtx +} + +// applyExtensions decorates a stub component holding the given objects with the +// extension registered under key in s, then renders it - returning the decorated +// create and delete lists. +func applyExtensions(s *extensions.Set, key string, ctx extensions.RenderContext, create, del []client.Object) ([]client.Object, []client.Object) { + stub := stubExtComponent{key: key, extCtx: ctx.Component, create: create, delete: del} + return s.Decorate(stub, ctx).Objects() +} diff --git a/pkg/enterprise/guardian.go b/pkg/enterprise/guardian.go index cff8185428..0d81779c8f 100644 --- a/pkg/enterprise/guardian.go +++ b/pkg/enterprise/guardian.go @@ -39,10 +39,10 @@ import ( ) func registerGuardian(s *extensions.Set) { - s.Register(operatorv1.CalicoEnterprise, render.GuardianName, extensions.Extension{ + s.Register(operatorv1.CalicoEnterprise, render.GuardianName, extensions.ComponentExtension{ Modify: modifyGuardian, }) - s.Register(operatorv1.CalicoEnterprise, render.ComponentNameGuardianPolicy, extensions.Extension{ + s.Register(operatorv1.CalicoEnterprise, render.ComponentNameGuardianPolicy, extensions.ComponentExtension{ Modify: modifyGuardianPolicy, }) } diff --git a/pkg/enterprise/guardian_test.go b/pkg/enterprise/guardian_test.go index 5223f72a47..87f5207ee2 100644 --- a/pkg/enterprise/guardian_test.go +++ b/pkg/enterprise/guardian_test.go @@ -57,7 +57,7 @@ var _ = Describe("guardian enterprise modifier", func() { } It("appends the secrets RBAC and UI settings", func() { - out, _ := ext.ApplyModifiers(render.GuardianName, ctxWith(render.GuardianExtensionContext{}), newObjs(), nil) + out, _ := applyExtensions(ext, render.GuardianName, ctxWith(render.GuardianExtensionContext{}), newObjs(), nil) _, ok := extensions.FindObject[*rbacv1.Role](out, render.GuardianSecretsRole) Expect(ok).To(BeTrue()) _, ok = extensions.FindObject[*rbacv1.RoleBinding](out, render.GuardianSecretsRoleBindingName) @@ -67,7 +67,7 @@ var _ = Describe("guardian enterprise modifier", func() { }) It("adds the elasticsearch and kibana service ports", func() { - out, _ := ext.ApplyModifiers(render.GuardianName, ctxWith(render.GuardianExtensionContext{}), newObjs(), nil) + out, _ := applyExtensions(ext, render.GuardianName, ctxWith(render.GuardianExtensionContext{}), newObjs(), nil) svc, _ := extensions.FindObject[*corev1.Service](out, render.GuardianServiceName) names := []string{} for _, p := range svc.Spec.Ports { @@ -80,7 +80,7 @@ var _ = Describe("guardian enterprise modifier", func() { gc := render.GuardianExtensionContext{ Impersonation: &operatorv1.Impersonation{Users: []string{"foo"}, Groups: []string{"bar"}}, } - out, _ := ext.ApplyModifiers(render.GuardianName, ctxWith(gc), newObjs(), nil) + out, _ := applyExtensions(ext, render.GuardianName, ctxWith(gc), newObjs(), nil) role, _ := extensions.FindObject[*rbacv1.ClusterRole](out, render.GuardianClusterRoleName) // The single OSS placeholder rule is gone, replaced by the enterprise set. @@ -91,14 +91,14 @@ var _ = Describe("guardian enterprise modifier", func() { It("adds the CA bundle env to the guardian container", func() { gc := render.GuardianExtensionContext{TrustedBundleMountPath: "/ca/bundle"} - out, _ := ext.ApplyModifiers(render.GuardianName, ctxWith(gc), newObjs(), nil) + out, _ := applyExtensions(ext, render.GuardianName, ctxWith(gc), newObjs(), nil) dep, _ := extensions.FindObject[*appsv1.Deployment](out, render.GuardianDeploymentName) Expect(dep.Spec.Template.Spec.Containers[0].Env).To(ContainElement(corev1.EnvVar{Name: "GUARDIAN_PROMETHEUS_CA_BUNDLE_PATH", Value: "/ca/bundle"})) }) It("does nothing for the Calico variant", func() { ctx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}} - out, _ := ext.ApplyModifiers(render.GuardianName, ctx, newObjs(), nil) + out, _ := applyExtensions(ext, render.GuardianName, ctx, newObjs(), nil) Expect(out).To(HaveLen(len(newObjs()))) role, _ := extensions.FindObject[*rbacv1.ClusterRole](out, render.GuardianClusterRoleName) Expect(role.Rules).To(Equal([]rbacv1.PolicyRule{{Verbs: []string{"get"}}})) diff --git a/pkg/enterprise/node.go b/pkg/enterprise/node.go index 752155bd01..d0ca3e2135 100644 --- a/pkg/enterprise/node.go +++ b/pkg/enterprise/node.go @@ -47,7 +47,7 @@ const ( ) func registerNode(s *extensions.Set) { - s.Register(operatorv1.CalicoEnterprise, render.ComponentNameNode, extensions.Extension{ + s.Register(operatorv1.CalicoEnterprise, render.ComponentNameNode, extensions.ComponentExtension{ Image: func(in *operatorv1.InstallationSpec) components.Component { return components.ComponentTigeraNode }, @@ -56,7 +56,7 @@ func registerNode(s *extensions.Set) { // The node component renders the cni-plugins init container; its image // resolves through its own override key. - s.Register(operatorv1.CalicoEnterprise, render.ComponentNameCNIPlugins, extensions.Extension{ + s.Register(operatorv1.CalicoEnterprise, render.ComponentNameCNIPlugins, extensions.ComponentExtension{ Image: func(in *operatorv1.InstallationSpec) components.Component { return components.ComponentTigeraCNIPlugins }, diff --git a/pkg/enterprise/node_test.go b/pkg/enterprise/node_test.go index 23ca58c98e..4257766122 100644 --- a/pkg/enterprise/node_test.go +++ b/pkg/enterprise/node_test.go @@ -81,7 +81,7 @@ var _ = Describe("node enterprise modifier", func() { } It("adds the enterprise cluster role rules", func() { - out, _ := ext.ApplyModifiers(render.ComponentNameNode, entCtx(), newObjs(), nil) + out, _ := applyExtensions(ext, render.ComponentNameNode, entCtx(), newObjs(), nil) nodeRole, ok := extensions.FindObject[*rbacv1.ClusterRole](out, render.CalicoNodeObjectName) Expect(ok).To(BeTrue()) @@ -93,7 +93,7 @@ var _ = Describe("node enterprise modifier", func() { }) It("adds the enterprise felix env to the node container", func() { - out, _ := ext.ApplyModifiers(render.ComponentNameNode, entCtx(), newObjs(), nil) + out, _ := applyExtensions(ext, render.ComponentNameNode, entCtx(), newObjs(), nil) ds, _ := extensions.FindObject[*appsv1.DaemonSet](out, common.NodeDaemonSetName) c := nodeContainer(ds) @@ -110,13 +110,13 @@ var _ = Describe("node enterprise modifier", func() { ctx := entCtx() ctx.FelixConfiguration = &v3.FelixConfiguration{Spec: v3.FelixConfigurationSpec{PrometheusReporterPort: &reporter}} - out, _ := ext.ApplyModifiers(render.ComponentNameNode, ctx, newObjs(), nil) + out, _ := applyExtensions(ext, render.ComponentNameNode, ctx, newObjs(), nil) ds, _ := extensions.FindObject[*appsv1.DaemonSet](out, common.NodeDaemonSetName) Expect(nodeContainer(ds).Env).To(ContainElement(corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERPORT", Value: "7081"})) }) It("appends the BGP metrics readiness check when the bird check is present", func() { - out, _ := ext.ApplyModifiers(render.ComponentNameNode, entCtx(), newObjs(), nil) + out, _ := applyExtensions(ext, render.ComponentNameNode, entCtx(), newObjs(), nil) ds, _ := extensions.FindObject[*appsv1.DaemonSet](out, common.NodeDaemonSetName) Expect(nodeContainer(ds).ReadinessProbe.Exec.Command).To(ContainElement("--bgp-metrics-ready")) }) @@ -126,7 +126,7 @@ var _ = Describe("node enterprise modifier", func() { ds := objs[2].(*appsv1.DaemonSet) ds.Spec.Template.Spec.Containers[0].ReadinessProbe.Exec.Command = []string{"/bin/calico-node", "--felix-ready"} - out, _ := ext.ApplyModifiers(render.ComponentNameNode, entCtx(), objs, nil) + out, _ := applyExtensions(ext, render.ComponentNameNode, entCtx(), objs, nil) got, _ := extensions.FindObject[*appsv1.DaemonSet](out, common.NodeDaemonSetName) Expect(nodeContainer(got).ReadinessProbe.Exec.Command).NotTo(ContainElement("--bgp-metrics-ready")) }) @@ -136,7 +136,7 @@ var _ = Describe("node enterprise modifier", func() { ctx := entCtx() ctx.Installation.CalicoNetwork = &operatorv1.CalicoNetworkSpec{MultiInterfaceMode: &mode} - out, _ := ext.ApplyModifiers(render.ComponentNameNode, ctx, newObjs(), nil) + out, _ := applyExtensions(ext, render.ComponentNameNode, ctx, newObjs(), nil) ds, _ := extensions.FindObject[*appsv1.DaemonSet](out, common.NodeDaemonSetName) want := corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: mode.Value()} @@ -145,7 +145,7 @@ var _ = Describe("node enterprise modifier", func() { }) It("appends the node metrics service", func() { - out, _ := ext.ApplyModifiers(render.ComponentNameNode, entCtx(), newObjs(), nil) + out, _ := applyExtensions(ext, render.ComponentNameNode, entCtx(), newObjs(), nil) svc, ok := extensions.FindObject[*corev1.Service](out, render.CalicoNodeMetricsService) Expect(ok).To(BeTrue()) Expect(svc.Spec.Ports).To(HaveLen(2)) @@ -164,7 +164,7 @@ var _ = Describe("node enterprise modifier", func() { PrometheusMetricsEnabled: &enabled, }} - out, _ := ext.ApplyModifiers(render.ComponentNameNode, ctx, newObjs(), nil) + out, _ := applyExtensions(ext, render.ComponentNameNode, ctx, newObjs(), nil) svc, _ := extensions.FindObject[*corev1.Service](out, render.CalicoNodeMetricsService) Expect(svc.Spec.Ports).To(HaveLen(3)) Expect(svc.Spec.Ports[0].Port).To(Equal(int32(7081))) @@ -174,7 +174,7 @@ var _ = Describe("node enterprise modifier", func() { It("is a no-op for the Calico variant", func() { ctx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}} - out, _ := ext.ApplyModifiers(render.ComponentNameNode, ctx, newObjs(), nil) + out, _ := applyExtensions(ext, render.ComponentNameNode, ctx, newObjs(), nil) _, ok := extensions.FindObject[*corev1.Service](out, render.CalicoNodeMetricsService) Expect(ok).To(BeFalse()) @@ -183,7 +183,7 @@ var _ = Describe("node enterprise modifier", func() { }) It("does not panic on a zero RenderContext", func() { - out, _ := ext.ApplyModifiers(render.ComponentNameNode, extensions.RenderContext{}, newObjs(), nil) + out, _ := applyExtensions(ext, render.ComponentNameNode, extensions.RenderContext{}, newObjs(), nil) _, ok := extensions.FindObject[*corev1.Service](out, render.CalicoNodeMetricsService) Expect(ok).To(BeFalse()) }) diff --git a/pkg/enterprise/typha.go b/pkg/enterprise/typha.go index 4571450365..ce3a434bbe 100644 --- a/pkg/enterprise/typha.go +++ b/pkg/enterprise/typha.go @@ -26,7 +26,7 @@ import ( ) func registerTypha(s *extensions.Set) { - s.Register(operatorv1.CalicoEnterprise, render.ComponentNameTypha, extensions.Extension{ + s.Register(operatorv1.CalicoEnterprise, render.ComponentNameTypha, extensions.ComponentExtension{ Modify: modifyTypha, }) } diff --git a/pkg/enterprise/typha_test.go b/pkg/enterprise/typha_test.go index 2b1211c4ec..cfc60b2b62 100644 --- a/pkg/enterprise/typha_test.go +++ b/pkg/enterprise/typha_test.go @@ -49,7 +49,7 @@ var _ = Describe("typha enterprise modifier", func() { Variant: operatorv1.CalicoEnterprise, CalicoNetwork: &operatorv1.CalicoNetworkSpec{MultiInterfaceMode: &multiMode}, }} - out, _ := ext.ApplyModifiers(render.ComponentNameTypha, ctx, newObjs(), nil) + out, _ := applyExtensions(ext, render.ComponentNameTypha, ctx, newObjs(), nil) role := out[0].(*rbacv1.ClusterRole) Expect(role.Rules).To(ContainElement(HaveField("Resources", ContainElement("licensekeys")))) @@ -69,14 +69,14 @@ var _ = Describe("typha enterprise modifier", func() { Variant: operatorv1.Calico, CalicoNetwork: &operatorv1.CalicoNetworkSpec{MultiInterfaceMode: &multiMode}, }} - out, _ := ext.ApplyModifiers(render.ComponentNameTypha, ctx, newObjs(), nil) + out, _ := applyExtensions(ext, render.ComponentNameTypha, ctx, newObjs(), nil) Expect(out[0].(*rbacv1.ClusterRole).Rules).To(BeEmpty()) dep := out[1].(*appsv1.Deployment) Expect(dep.Spec.Template.Spec.Containers[0].Env).To(BeEmpty()) }) It("does not panic on a zero Context (nil Installation)", func() { - out, _ := ext.ApplyModifiers(render.ComponentNameTypha, extensions.RenderContext{}, newObjs(), nil) + out, _ := applyExtensions(ext, render.ComponentNameTypha, extensions.RenderContext{}, newObjs(), nil) Expect(out[0].(*rbacv1.ClusterRole).Rules).To(BeEmpty()) }) }) diff --git a/pkg/enterprise/windows.go b/pkg/enterprise/windows.go index f4674323b3..7883429e41 100644 --- a/pkg/enterprise/windows.go +++ b/pkg/enterprise/windows.go @@ -37,13 +37,13 @@ import ( var windowsNodeContainers = map[string]bool{"felix": true, "node": true, "confd": true} func registerWindows(s *extensions.Set) { - s.Register(operatorv1.CalicoEnterprise, render.ComponentNameWindowsNodeImg, extensions.Extension{ + s.Register(operatorv1.CalicoEnterprise, render.ComponentNameWindowsNodeImg, extensions.ComponentExtension{ Image: func(*operatorv1.InstallationSpec) components.Component { return components.ComponentTigeraNodeWindows }, }) - s.Register(operatorv1.CalicoEnterprise, render.ComponentNameWindowsCNIImg, extensions.Extension{ + s.Register(operatorv1.CalicoEnterprise, render.ComponentNameWindowsCNIImg, extensions.ComponentExtension{ Image: func(*operatorv1.InstallationSpec) components.Component { return components.ComponentTigeraCNIWindows }, }) - s.Register(operatorv1.CalicoEnterprise, render.ComponentNameWindows, extensions.Extension{ + s.Register(operatorv1.CalicoEnterprise, render.ComponentNameWindows, extensions.ComponentExtension{ Modify: modifyWindows, }) } diff --git a/pkg/enterprise/windows_test.go b/pkg/enterprise/windows_test.go index 9bc08e1d52..b6c6b0c2d0 100644 --- a/pkg/enterprise/windows_test.go +++ b/pkg/enterprise/windows_test.go @@ -97,7 +97,7 @@ var _ = Describe("windows enterprise modifier", func() { } It("appends the node-metrics service", func() { - out, _ := ext.ApplyModifiers(render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone, nil, nil), newObjs(), nil) + out, _ := applyExtensions(ext, render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone, nil, nil), newObjs(), nil) svc, ok := extensions.FindObject[*corev1.Service](out, render.WindowsNodeMetricsService) Expect(ok).To(BeTrue()) Expect(svc.Namespace).To(Equal(common.CalicoNamespace)) @@ -105,7 +105,7 @@ var _ = Describe("windows enterprise modifier", func() { }) It("swaps the cni log mount for the calico log volume and adds enterprise env", func() { - out, _ := ext.ApplyModifiers(render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone, nil, nil), newObjs(), nil) + out, _ := applyExtensions(ext, render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone, nil, nil), newObjs(), nil) d := ds(out) Expect(d.Spec.Template.Spec.Volumes).To(ContainElement(HaveField("Name", "var-log-calico"))) @@ -122,7 +122,7 @@ var _ = Describe("windows enterprise modifier", func() { }) It("sets the trusted DNS server on openshift", func() { - out, _ := ext.ApplyModifiers(render.ComponentNameWindows, ctxFor(operatorv1.ProviderOpenShift, nil, nil), newObjs(), nil) + out, _ := applyExtensions(ext, render.ComponentNameWindows, ctxFor(operatorv1.ProviderOpenShift, nil, nil), newObjs(), nil) Expect(container(ds(out), "node").Env).To(ContainElement(corev1.EnvVar{Name: "FELIX_DNSTRUSTEDSERVERS", Value: "k8s-service:openshift-dns/dns-default"})) }) @@ -136,7 +136,7 @@ var _ = Describe("windows enterprise modifier", func() { Expect(err).NotTo(HaveOccurred()) bundle := cm.CreateTrustedBundle() - out, _ := ext.ApplyModifiers(render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone, tls, bundle), newObjs(), nil) + out, _ := applyExtensions(ext, render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone, tls, bundle), newObjs(), nil) d := ds(out) Expect(d.Spec.Template.Spec.Volumes).To(ContainElement(tls.Volume())) @@ -147,7 +147,7 @@ var _ = Describe("windows enterprise modifier", func() { It("does nothing for the Calico variant", func() { ctx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}} - out, _ := ext.ApplyModifiers(render.ComponentNameWindows, ctx, newObjs(), nil) + out, _ := applyExtensions(ext, render.ComponentNameWindows, ctx, newObjs(), nil) _, ok := extensions.FindObject[*corev1.Service](out, render.WindowsNodeMetricsService) Expect(ok).To(BeFalse()) Expect(ds(out).Spec.Template.Spec.Volumes).To(BeEmpty()) diff --git a/pkg/extensions/decorate_helpers_test.go b/pkg/extensions/decorate_helpers_test.go new file mode 100644 index 0000000000..227c4f552a --- /dev/null +++ b/pkg/extensions/decorate_helpers_test.go @@ -0,0 +1,65 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extensions_test + +import ( + client "sigs.k8s.io/controller-runtime/pkg/client" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/extensions" + rmeta "github.com/tigera/operator/pkg/render/common/meta" +) + +// stubExtComponent adapts raw object lists to a render.Component so a registered +// extension can be exercised through Set.Decorate, the same seam the component +// handler uses. key selects the extension; extCtx is delivered as the per-component +// ExtensionContext. +type stubExtComponent struct { + key string + extCtx any + create, delete []client.Object +} + +func (s stubExtComponent) ResolveImages(*operatorv1.ImageSet) error { + return nil +} + +func (s stubExtComponent) Objects() ([]client.Object, []client.Object) { + return s.create, s.delete +} + +func (s stubExtComponent) Ready() bool { + return true +} + +func (s stubExtComponent) SupportedOSType() rmeta.OSType { + return rmeta.OSTypeAny +} + +func (s stubExtComponent) ModifierKey() string { + return s.key +} + +func (s stubExtComponent) ExtensionContext() any { + return s.extCtx +} + +// applyExtensions decorates a stub component holding the given objects with the +// extension registered under key in s, then renders it - returning the decorated +// create and delete lists. +func applyExtensions(s *extensions.Set, key string, ctx extensions.RenderContext, create, del []client.Object) ([]client.Object, []client.Object) { + stub := stubExtComponent{key: key, extCtx: ctx.Component, create: create, delete: del} + return s.Decorate(stub, ctx).Objects() +} diff --git a/pkg/extensions/extension.go b/pkg/extensions/extension.go index 346e7347f2..e9ef998a6b 100644 --- a/pkg/extensions/extension.go +++ b/pkg/extensions/extension.go @@ -18,14 +18,15 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/render" ) -// Extension is everything a variant layers onto one render component. Every -// field is optional: a component that only needs a different image sets Image -// and leaves Modify nil, and vice versa. This is the single registration a +// ComponentExtension is everything a variant layers onto one render component. +// Every field is optional: a component that only needs a different image sets +// Image and leaves Modify nil, and vice versa. This is the single registration a // variant makes per component, so all of that component's variance lives in one // place. -type Extension struct { +type ComponentExtension struct { // Image overrides the component's image. Resolved during ResolveImages, in // the render package, via the imageoverride leaf. Image ImageOverride @@ -47,19 +48,45 @@ type modifierKey struct { component string } -// ApplyModifiers runs the modifier registered for the named component and the -// installation's variant over the create and delete lists, returning them -// unchanged when none is registered (or when no installation is set). Safe to -// call on a nil Set, which is a no-op - the core operator registers no -// modifiers. -func (s *Set) ApplyModifiers(component string, ctx RenderContext, create, delete []client.Object) ([]client.Object, []client.Object) { +// Decorate wraps component with the variant extension registered for it, so +// that when the handler renders the component its objects are post-processed by +// the modifier registered for the component and the installation's variant. A +// decorated component is itself a render.Component, so it flows through the +// component handler exactly like any other. Returns component unchanged when it +// exposes no extension point, when no modifier is registered for it, when no +// installation is set, or on a nil Set (the core operator registers no +// extensions). +func (s *Set) Decorate(component render.Component, ctx RenderContext) render.Component { if s == nil || ctx.Installation == nil { - return create, delete + return component } - if fn, ok := s.modifiers[modifierKey{ctx.Installation.Variant, component}]; ok { - create, delete = fn(ctx, create, delete) + ext, ok := component.(render.Extensible) + if !ok { + return component } - return create, delete + modify, ok := s.modifiers[modifierKey{ctx.Installation.Variant, ext.ModifierKey()}] + if !ok { + return component + } + if p, ok := component.(render.ExtensionContextProvider); ok { + ctx.Component = p.ExtensionContext() + } + return &decoratedComponent{Component: component, ctx: ctx, modify: modify} +} + +// decoratedComponent is the render.Component produced by Decorate: it renders +// its embedded base component and then runs the variant modifier over the +// result. It embeds the base render.Component, so ResolveImages, SupportedOSType, +// and Ready delegate to the base; only Objects is augmented. +type decoratedComponent struct { + render.Component + ctx RenderContext + modify Modifier +} + +func (d *decoratedComponent) Objects() ([]client.Object, []client.Object) { + create, del := d.Component.Objects() + return d.modify(d.ctx, create, del) } // FindObject returns the first object of type T with the given name. diff --git a/pkg/extensions/extension_test.go b/pkg/extensions/extension_test.go index b02ec3d24d..8cbb0a7c89 100644 --- a/pkg/extensions/extension_test.go +++ b/pkg/extensions/extension_test.go @@ -34,7 +34,7 @@ var _ = Describe("extension registry", func() { entCtx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}} It("applies a registered modifier to the matching component and variant", func() { - s.Register(operatorv1.CalicoEnterprise, "test", extensions.Extension{ + s.Register(operatorv1.CalicoEnterprise, "test", extensions.ComponentExtension{ Modify: func(ctx extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { cm, ok := extensions.FindObject[*corev1.ConfigMap](objs, "cm") Expect(ok).To(BeTrue()) @@ -44,7 +44,7 @@ var _ = Describe("extension registry", func() { }) in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} - out, _ := s.ApplyModifiers("test", entCtx, in, nil) + out, _ := applyExtensions(s, "test", entCtx, in, nil) Expect(out).To(HaveLen(2)) cm := out[0].(*corev1.ConfigMap) @@ -53,14 +53,14 @@ var _ = Describe("extension registry", func() { }) It("lets a modifier append to the delete list", func() { - s.Register(operatorv1.CalicoEnterprise, "test", extensions.Extension{ + s.Register(operatorv1.CalicoEnterprise, "test", extensions.ComponentExtension{ Modify: func(_ extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { return objs, append(del, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "stale"}}) }, }) in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} - out, del := s.ApplyModifiers("test", entCtx, in, nil) + out, del := applyExtensions(s, "test", entCtx, in, nil) Expect(out).To(Equal(in)) Expect(del).To(HaveLen(1)) Expect(del[0].GetName()).To(Equal("stale")) @@ -68,12 +68,12 @@ var _ = Describe("extension registry", func() { It("returns objects unchanged when no modifier is registered", func() { in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} - out, _ := s.ApplyModifiers("unregistered", entCtx, in, nil) + out, _ := applyExtensions(s, "unregistered", entCtx, in, nil) Expect(out).To(Equal(in)) }) It("does not apply a modifier registered for a different variant", func() { - s.Register(operatorv1.CalicoEnterprise, "test", extensions.Extension{ + s.Register(operatorv1.CalicoEnterprise, "test", extensions.ComponentExtension{ Modify: func(_ extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { return append(objs, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "extra"}}), del }, @@ -81,19 +81,19 @@ var _ = Describe("extension registry", func() { calicoCtx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}} in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} - out, _ := s.ApplyModifiers("test", calicoCtx, in, nil) + out, _ := applyExtensions(s, "test", calicoCtx, in, nil) Expect(out).To(Equal(in)) }) It("returns objects unchanged when no installation is set", func() { in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} - out, _ := s.ApplyModifiers("test", extensions.RenderContext{}, in, nil) + out, _ := applyExtensions(s, "test", extensions.RenderContext{}, in, nil) Expect(out).To(Equal(in)) }) It("replaces rather than stacks when a (variant, component) is registered twice", func() { - add := func(name string) extensions.Extension { - return extensions.Extension{ + add := func(name string) extensions.ComponentExtension { + return extensions.ComponentExtension{ Modify: func(_ extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { return append(objs, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: name}}), del }, @@ -102,7 +102,7 @@ var _ = Describe("extension registry", func() { s.Register(operatorv1.CalicoEnterprise, "test", add("first")) s.Register(operatorv1.CalicoEnterprise, "test", add("second")) - out, _ := s.ApplyModifiers("test", entCtx, nil, nil) + out, _ := applyExtensions(s, "test", entCtx, nil, nil) Expect(out).To(HaveLen(1)) Expect(out[0].GetName()).To(Equal("second")) }) diff --git a/pkg/extensions/image_test.go b/pkg/extensions/image_test.go index 0581d8cc08..dcc4876fdd 100644 --- a/pkg/extensions/image_test.go +++ b/pkg/extensions/image_test.go @@ -27,7 +27,7 @@ var _ = Describe("image overrides", func() { var s *extensions.Set BeforeEach(func() { s = extensions.NewSet() - s.Register(operatorv1.CalicoEnterprise, "node", extensions.Extension{ + s.Register(operatorv1.CalicoEnterprise, "node", extensions.ComponentExtension{ Image: func(in *operatorv1.InstallationSpec) components.Component { return components.ComponentTigeraNode }, diff --git a/pkg/extensions/set.go b/pkg/extensions/set.go index b638622750..08aa52f259 100644 --- a/pkg/extensions/set.go +++ b/pkg/extensions/set.go @@ -28,7 +28,7 @@ import ( // registries, so nothing is wired by import side effect. // // The zero value is not usable; build one with NewSet. The methods that read it -// (BuildContext, ApplyModifiers, ResolveImage, Images) are nil-safe so the core +// (BuildContext, Decorate, ResolveImage, Images) are nil-safe so the core // operator can pass a nil Set and get base behavior. type Set struct { setups map[operatorv1.ProductVariant]Setup @@ -49,7 +49,7 @@ func NewSet() *Set { // variant. A (variant, component) pair has at most one extension; registration // replaces any prior one. The image override and the modifier are stored // separately, so a component can set either field or both. -func (s *Set) Register(variant operatorv1.ProductVariant, component string, e Extension) { +func (s *Set) Register(variant operatorv1.ProductVariant, component string, e ComponentExtension) { if e.Image != nil { s.images.Register(variant, component, e.Image) } diff --git a/pkg/render/apiserver_test.go b/pkg/render/apiserver_test.go index a81ebb6e01..31b15779ee 100644 --- a/pkg/render/apiserver_test.go +++ b/pkg/render/apiserver_test.go @@ -73,7 +73,7 @@ func apiServerObjects(c render.Component) ([]client.Object, []client.Object) { rc.Installation = ec.Config.Installation rc.Component = ec } - return ext.ApplyModifiers(render.ComponentNameAPIServer, rc, create, del) + return applyExtensions(ext, render.ComponentNameAPIServer, rc, create, del) } var _ = Describe("API server rendering tests (Calico Enterprise)", func() { diff --git a/pkg/render/decorate_helpers_test.go b/pkg/render/decorate_helpers_test.go new file mode 100644 index 0000000000..750abb8df4 --- /dev/null +++ b/pkg/render/decorate_helpers_test.go @@ -0,0 +1,65 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package render_test + +import ( + client "sigs.k8s.io/controller-runtime/pkg/client" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/extensions" + rmeta "github.com/tigera/operator/pkg/render/common/meta" +) + +// stubExtComponent adapts raw object lists to a render.Component so a registered +// extension can be exercised through Set.Decorate, the same seam the component +// handler uses. key selects the extension; extCtx is delivered as the per-component +// ExtensionContext. +type stubExtComponent struct { + key string + extCtx any + create, delete []client.Object +} + +func (s stubExtComponent) ResolveImages(*operatorv1.ImageSet) error { + return nil +} + +func (s stubExtComponent) Objects() ([]client.Object, []client.Object) { + return s.create, s.delete +} + +func (s stubExtComponent) Ready() bool { + return true +} + +func (s stubExtComponent) SupportedOSType() rmeta.OSType { + return rmeta.OSTypeAny +} + +func (s stubExtComponent) ModifierKey() string { + return s.key +} + +func (s stubExtComponent) ExtensionContext() any { + return s.extCtx +} + +// applyExtensions decorates a stub component holding the given objects with the +// extension registered under key in s, then renders it - returning the decorated +// create and delete lists. +func applyExtensions(s *extensions.Set, key string, ctx extensions.RenderContext, create, del []client.Object) ([]client.Object, []client.Object) { + stub := stubExtComponent{key: key, extCtx: ctx.Component, create: create, delete: del} + return s.Decorate(stub, ctx).Objects() +} diff --git a/pkg/render/guardian_test.go b/pkg/render/guardian_test.go index 33c04099fd..44e4e2fe4a 100644 --- a/pkg/render/guardian_test.go +++ b/pkg/render/guardian_test.go @@ -54,7 +54,7 @@ func guardianObjects(cfg *render.GuardianConfiguration) []client.Object { if p, ok := g.(render.ExtensionContextProvider); ok { rc.Component = p.ExtensionContext() } - out, _ := ext.ApplyModifiers(render.GuardianName, rc, objs, nil) + out, _ := applyExtensions(ext, render.GuardianName, rc, objs, nil) return out } @@ -116,7 +116,7 @@ var _ = Describe("Rendering tests", func() { if p, ok := g.(render.ExtensionContextProvider); ok { rc.Component = p.ExtensionContext() } - resources, _ = ext.ApplyModifiers(render.GuardianName, rc, resources, nil) + resources, _ = applyExtensions(ext, render.GuardianName, rc, resources, nil) } BeforeEach(func() { @@ -352,7 +352,7 @@ var _ = Describe("Rendering tests", func() { if p, ok := g.(render.ExtensionContextProvider); ok { rc.Component = p.ExtensionContext() } - resources, _ = ext.ApplyModifiers(render.ComponentNameGuardianPolicy, rc, objs, nil) + resources, _ = applyExtensions(ext, render.ComponentNameGuardianPolicy, rc, objs, nil) } Context("policy rendering based on variant and IncludeEgressNetworkPolicy", func() { diff --git a/pkg/render/node_enterprise_test.go b/pkg/render/node_enterprise_test.go index 7b11bbf6eb..c9c32b69fd 100644 --- a/pkg/render/node_enterprise_test.go +++ b/pkg/render/node_enterprise_test.go @@ -110,7 +110,7 @@ var _ = Describe("node enterprise modifier integration", func() { comp := render.Node(cfg) Expect(comp.ResolveImages(nil)).NotTo(HaveOccurred()) objs, _ := comp.Objects() - out, _ := ext.ApplyModifiers(render.ComponentNameNode, renderCtx, objs, nil) + out, _ := applyExtensions(ext, render.ComponentNameNode, renderCtx, objs, nil) return out } @@ -167,7 +167,7 @@ var _ = Describe("node enterprise modifier integration", func() { }) Expect(comp.ResolveImages(nil)).NotTo(HaveOccurred()) objs, _ := comp.Objects() - objs, _ = ext.ApplyModifiers(render.ComponentNameTypha, renderCtx, objs, nil) + objs, _ = applyExtensions(ext, render.ComponentNameTypha, renderCtx, objs, nil) role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, "calico-typha") Expect(ok).To(BeTrue()) diff --git a/pkg/render/windows_test.go b/pkg/render/windows_test.go index e92bc86478..f15e185eda 100644 --- a/pkg/render/windows_test.go +++ b/pkg/render/windows_test.go @@ -54,7 +54,7 @@ func renderWindows(cfg *render.WindowsConfiguration) []client.Object { if p, ok := comp.(render.ExtensionContextProvider); ok { rc.Component = p.ExtensionContext() } - out, _ := ext.ApplyModifiers(render.ComponentNameWindows, rc, objs, nil) + out, _ := applyExtensions(ext, render.ComponentNameWindows, rc, objs, nil) return out } From 8ef2965b65e710debec4c586bc1d827112d159a0 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Thu, 18 Jun 2026 10:33:07 -0700 Subject: [PATCH 29/38] Restructure extensions into per-variant bundles with a typed controller hook A Set holds one Variant bundle per product variant; the controller selects one from the installation variant, so each component has at most one extension and a modifier never re-checks the variant. ControllerExtension (Validate, ExtendContext) replaces the setup func as the controller-side hook. ControllerContext embeds RenderContext and carries the cluster-access deps, so a modifier given a RenderContext can't do I/O. Component modifiers register with RegisterModifier, which hands them typed config and removes RenderContext.Component. --- cmd/main.go | 6 +- .../installation/core_controller.go | 19 ++- pkg/controller/utils/component_test.go | 10 +- pkg/enterprise/apiserver.go | 34 ++--- pkg/enterprise/decorate_helpers_test.go | 16 +- pkg/enterprise/guardian.go | 27 +--- pkg/enterprise/guardian_test.go | 15 +- pkg/enterprise/installation.go | 43 +++--- pkg/enterprise/installation_test.go | 27 ++-- pkg/enterprise/node.go | 16 +- pkg/enterprise/register.go | 34 +++-- pkg/enterprise/typha.go | 7 +- pkg/enterprise/windows.go | 23 +-- pkg/enterprise/windows_test.go | 23 +-- pkg/extensions/controllerextension.go | 58 ++++++++ pkg/extensions/controllerextension_test.go | 104 +++++++++++++ pkg/extensions/decorate_helpers_test.go | 16 +- pkg/extensions/doc.go | 38 +++-- pkg/extensions/extension.go | 69 +-------- pkg/extensions/extension_test.go | 40 ++--- pkg/extensions/image_test.go | 6 +- pkg/extensions/rendercontext.go | 27 ++-- pkg/extensions/set.go | 97 ++++++++---- pkg/extensions/setup.go | 78 ---------- pkg/extensions/setup_test.go | 89 ----------- pkg/extensions/variant.go | 138 ++++++++++++++++++ pkg/render/apiserver_test.go | 5 +- pkg/render/decorate_helpers_test.go | 16 +- pkg/render/guardian_test.go | 15 +- pkg/render/windows_test.go | 5 +- 30 files changed, 593 insertions(+), 508 deletions(-) create mode 100644 pkg/extensions/controllerextension.go create mode 100644 pkg/extensions/controllerextension_test.go delete mode 100644 pkg/extensions/setup.go delete mode 100644 pkg/extensions/setup_test.go create mode 100644 pkg/extensions/variant.go diff --git a/cmd/main.go b/cmd/main.go index eb63d9b666..60560c9e46 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -522,11 +522,7 @@ If a value other than 'all' is specified, the first CRD with a prefix of the spe ElasticExternal: discovery.UseExternalElastic(bootConfig), UseV3CRDs: v3CRDs, APIDiscovery: apiDiscovery, - // Hand the operator the in-repo Calico Enterprise extensions (modifiers, - // image overrides, and the installation setup). After the monorepo split - // the core operator's main passes none and calico-private's main passes - // its own. - Extensions: enterprise.New(), + Extensions: enterprise.New(), } // Before we start any controllers, make sure our options are valid. diff --git a/pkg/controller/installation/core_controller.go b/pkg/controller/installation/core_controller.go index 08444f08e8..baeb11364e 100644 --- a/pkg/controller/installation/core_controller.go +++ b/pkg/controller/installation/core_controller.go @@ -1209,15 +1209,22 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile calicoVersion = components.EnterpriseRelease } - renderCtx, err := r.opts.Extensions.BuildContext(extensions.Inputs{ + cc := extensions.ControllerContext{ + RenderContext: extensions.RenderContext{ + Installation: &instance.Spec, + FelixConfiguration: felixConfiguration, + ClusterDomain: r.opts.ClusterDomain, + TrustedBundle: typhaNodeTLS.TrustedBundle, + }, Ctx: ctx, Client: r.client, - Installation: &instance.Spec, - FelixConfiguration: felixConfiguration, CertificateManager: certificateManager, - TrustedBundle: typhaNodeTLS.TrustedBundle, - ClusterDomain: r.opts.ClusterDomain, - }) + } + if err := r.opts.Extensions.Validate(cc); err != nil { + r.status.SetDegraded(operatorv1.ResourceValidationError, "Invalid installation configuration for variant", err, reqLogger) + return reconcile.Result{}, err + } + renderCtx, err := r.opts.Extensions.ExtendContext(cc) if err != nil { r.status.SetDegraded(operatorv1.ResourceCreateError, "Error preparing installation extension", err, reqLogger) return reconcile.Result{}, err diff --git a/pkg/controller/utils/component_test.go b/pkg/controller/utils/component_test.go index ec369e2ac3..776ef6b4a5 100644 --- a/pkg/controller/utils/component_test.go +++ b/pkg/controller/utils/component_test.go @@ -2580,12 +2580,10 @@ func (mc *mockClient) SubResource(subResource string) client.SubResourceClient { var _ = Describe("componentHandler modifier application", func() { It("applies registered modifiers to a named component before create", func() { ext := extensions.NewSet() - ext.Register(operatorv1.CalicoEnterprise, "fake", extensions.ComponentExtension{ - Modify: func(ctx extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { - cm := objs[0].(*corev1.ConfigMap) - cm.Data = map[string]string{"patched": "yes"} - return objs, del - }, + ext.Variant(operatorv1.CalicoEnterprise).Modify("fake", func(ctx extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { + cm := objs[0].(*corev1.ConfigMap) + cm.Data = map[string]string{"patched": "yes"} + return objs, del }) s := runtime.NewScheme() diff --git a/pkg/enterprise/apiserver.go b/pkg/enterprise/apiserver.go index 913391cb16..7a265877d1 100644 --- a/pkg/enterprise/apiserver.go +++ b/pkg/enterprise/apiserver.go @@ -18,7 +18,6 @@ import ( "fmt" "strings" - "github.com/sirupsen/logrus" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -27,7 +26,6 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" - operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/extensions" @@ -52,26 +50,21 @@ type apiServer struct { calicoImage string } -func registerAPIServer(s *extensions.Set) { - s.Register(operatorv1.CalicoEnterprise, render.ComponentNameAPIServer, extensions.ComponentExtension{ - Modify: modifyAPIServer, - }) - // When running Calico, clean up any Enterprise objects left behind by a prior - // Enterprise installation. - s.Register(operatorv1.Calico, render.ComponentNameAPIServer, extensions.ComponentExtension{ - Modify: cleanupAPIServer, - }) +func registerAPIServer(v *extensions.Variant) { + extensions.RegisterModifier(v, render.ComponentNameAPIServer, modifyAPIServer) +} + +// registerAPIServerCleanup registers, for the Calico variant, the cleanup that +// deletes the Enterprise API server objects left behind by a prior Enterprise +// installation. +func registerAPIServerCleanup(v *extensions.Variant) { + extensions.RegisterModifier(v, render.ComponentNameAPIServer, cleanupAPIServer) } // modifyAPIServer layers Calico Enterprise behavior onto the rendered API server objects: // the query server container and its volumes, audit logging on the aggregation API server // container, the Enterprise RBAC objects, and the query server port on the Service. -func modifyAPIServer(ctx extensions.RenderContext, create, del []client.Object) ([]client.Object, []client.Object) { - ec, ok := ctx.Component.(render.APIServerExtensionContext) - if !ok { - logrus.Errorf("BUG: apiserver modifier got %T, want render.APIServerExtensionContext; leaving objects unchanged", ctx.Component) - return create, del - } +func modifyAPIServer(ctx extensions.RenderContext, ec render.APIServerExtensionContext, create, del []client.Object) ([]client.Object, []client.Object) { c := &apiServer{cfg: ec.Config, calicoImage: ec.CalicoImage} if dep, ok := extensions.FindObject[*appsv1.Deployment](create, render.APIServerName); ok { @@ -150,12 +143,7 @@ func modifyAPIServer(ctx extensions.RenderContext, create, del []client.Object) // cleanupAPIServer deletes the Enterprise API server objects when running Calico, so a // cluster switched from Enterprise to Calico does not leave them behind. -func cleanupAPIServer(ctx extensions.RenderContext, create, del []client.Object) ([]client.Object, []client.Object) { - ec, ok := ctx.Component.(render.APIServerExtensionContext) - if !ok { - logrus.Errorf("BUG: apiserver cleanup got %T, want render.APIServerExtensionContext; leaving objects unchanged", ctx.Component) - return create, del - } +func cleanupAPIServer(ctx extensions.RenderContext, ec render.APIServerExtensionContext, create, del []client.Object) ([]client.Object, []client.Object) { c := &apiServer{cfg: ec.Config} del = append(del, c.tigeraAPIServerClusterRole(), c.tigeraAPIServerClusterRoleBinding()) diff --git a/pkg/enterprise/decorate_helpers_test.go b/pkg/enterprise/decorate_helpers_test.go index 5bade86c8f..184a801a71 100644 --- a/pkg/enterprise/decorate_helpers_test.go +++ b/pkg/enterprise/decorate_helpers_test.go @@ -24,8 +24,8 @@ import ( // stubExtComponent adapts raw object lists to a render.Component so a registered // extension can be exercised through Set.Decorate, the same seam the component -// handler uses. key selects the extension; extCtx is delivered as the per-component -// ExtensionContext. +// handler uses. key selects the extension; extCtx is delivered as the component's +// ExtensionContext (the typed config a RegisterModifier modifier reads). type stubExtComponent struct { key string extCtx any @@ -57,9 +57,15 @@ func (s stubExtComponent) ExtensionContext() any { } // applyExtensions decorates a stub component holding the given objects with the -// extension registered under key in s, then renders it - returning the decorated -// create and delete lists. +// extension registered under key, then renders it. For a modifier that needs the +// component's typed config, use applyExtensionsWithContext. func applyExtensions(s *extensions.Set, key string, ctx extensions.RenderContext, create, del []client.Object) ([]client.Object, []client.Object) { - stub := stubExtComponent{key: key, extCtx: ctx.Component, create: create, delete: del} + return applyExtensionsWithContext(s, key, ctx, nil, create, del) +} + +// applyExtensionsWithContext is applyExtensions for a modifier that reads the +// component's typed config: extCtx is delivered as the stub's ExtensionContext. +func applyExtensionsWithContext(s *extensions.Set, key string, ctx extensions.RenderContext, extCtx any, create, del []client.Object) ([]client.Object, []client.Object) { + stub := stubExtComponent{key: key, extCtx: extCtx, create: create, delete: del} return s.Decorate(stub, ctx).Objects() } diff --git a/pkg/enterprise/guardian.go b/pkg/enterprise/guardian.go index 0d81779c8f..11895b0b29 100644 --- a/pkg/enterprise/guardian.go +++ b/pkg/enterprise/guardian.go @@ -29,7 +29,6 @@ import ( v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" "github.com/tigera/api/pkg/lib/numorstring" - operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" @@ -38,26 +37,16 @@ import ( operatorurl "github.com/tigera/operator/pkg/url" ) -func registerGuardian(s *extensions.Set) { - s.Register(operatorv1.CalicoEnterprise, render.GuardianName, extensions.ComponentExtension{ - Modify: modifyGuardian, - }) - s.Register(operatorv1.CalicoEnterprise, render.ComponentNameGuardianPolicy, extensions.ComponentExtension{ - Modify: modifyGuardianPolicy, - }) +func registerGuardian(v *extensions.Variant) { + extensions.RegisterModifier(v, render.GuardianName, modifyGuardian) + extensions.RegisterModifier(v, render.ComponentNameGuardianPolicy, modifyGuardianPolicy) } // modifyGuardianPolicy replaces the core OSS guardian network policy with the // enterprise management-cluster policy. Building the enterprise egress rules can // fail (proxy URL parsing); on failure we drop the policy entirely, matching the // core behavior of omitting it rather than installing a partial policy. -func modifyGuardianPolicy(ctx extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { - gpc, ok := ctx.Component.(render.GuardianPolicyExtensionContext) - if !ok { - logrus.Errorf("BUG: guardian policy modifier got %T, want render.GuardianPolicyExtensionContext; leaving objects unchanged", ctx.Component) - return objs, del - } - +func modifyGuardianPolicy(ctx extensions.RenderContext, gpc render.GuardianPolicyExtensionContext, objs, del []client.Object) ([]client.Object, []client.Object) { policy, ok := extensions.FindObject[*v3.NetworkPolicy](objs, render.GuardianPolicyName) if !ok { return objs, del @@ -215,13 +204,7 @@ func enterpriseGuardianPolicySpec(gpc render.GuardianPolicyExtensionContext) (v3 // objects: the secrets Role/RoleBinding and default UI settings, the // elasticsearch/kibana service ports, the management-cluster-request cluster // role rules (which replace the OSS rules), and the CA bundle env vars. -func modifyGuardian(ctx extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { - gc, ok := ctx.Component.(render.GuardianExtensionContext) - if !ok { - logrus.Errorf("BUG: guardian modifier got %T, want render.GuardianExtensionContext; leaving objects unchanged", ctx.Component) - return objs, del - } - +func modifyGuardian(ctx extensions.RenderContext, gc render.GuardianExtensionContext, objs, del []client.Object) ([]client.Object, []client.Object) { if role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, render.GuardianClusterRoleName); ok { role.Rules = guardianEnterpriseRules(gc) } diff --git a/pkg/enterprise/guardian_test.go b/pkg/enterprise/guardian_test.go index 87f5207ee2..07e6694904 100644 --- a/pkg/enterprise/guardian_test.go +++ b/pkg/enterprise/guardian_test.go @@ -49,15 +49,10 @@ var _ = Describe("guardian enterprise modifier", func() { } } - ctxWith := func(c render.GuardianExtensionContext) extensions.RenderContext { - return extensions.RenderContext{ - Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}, - Component: c, - } - } + entCtx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}} It("appends the secrets RBAC and UI settings", func() { - out, _ := applyExtensions(ext, render.GuardianName, ctxWith(render.GuardianExtensionContext{}), newObjs(), nil) + out, _ := applyExtensionsWithContext(ext, render.GuardianName, entCtx, render.GuardianExtensionContext{}, newObjs(), nil) _, ok := extensions.FindObject[*rbacv1.Role](out, render.GuardianSecretsRole) Expect(ok).To(BeTrue()) _, ok = extensions.FindObject[*rbacv1.RoleBinding](out, render.GuardianSecretsRoleBindingName) @@ -67,7 +62,7 @@ var _ = Describe("guardian enterprise modifier", func() { }) It("adds the elasticsearch and kibana service ports", func() { - out, _ := applyExtensions(ext, render.GuardianName, ctxWith(render.GuardianExtensionContext{}), newObjs(), nil) + out, _ := applyExtensionsWithContext(ext, render.GuardianName, entCtx, render.GuardianExtensionContext{}, newObjs(), nil) svc, _ := extensions.FindObject[*corev1.Service](out, render.GuardianServiceName) names := []string{} for _, p := range svc.Spec.Ports { @@ -80,7 +75,7 @@ var _ = Describe("guardian enterprise modifier", func() { gc := render.GuardianExtensionContext{ Impersonation: &operatorv1.Impersonation{Users: []string{"foo"}, Groups: []string{"bar"}}, } - out, _ := applyExtensions(ext, render.GuardianName, ctxWith(gc), newObjs(), nil) + out, _ := applyExtensionsWithContext(ext, render.GuardianName, entCtx, gc, newObjs(), nil) role, _ := extensions.FindObject[*rbacv1.ClusterRole](out, render.GuardianClusterRoleName) // The single OSS placeholder rule is gone, replaced by the enterprise set. @@ -91,7 +86,7 @@ var _ = Describe("guardian enterprise modifier", func() { It("adds the CA bundle env to the guardian container", func() { gc := render.GuardianExtensionContext{TrustedBundleMountPath: "/ca/bundle"} - out, _ := applyExtensions(ext, render.GuardianName, ctxWith(gc), newObjs(), nil) + out, _ := applyExtensionsWithContext(ext, render.GuardianName, entCtx, gc, newObjs(), nil) dep, _ := extensions.FindObject[*appsv1.Deployment](out, render.GuardianDeploymentName) Expect(dep.Spec.Template.Spec.Containers[0].Env).To(ContainElement(corev1.EnvVar{Name: "GUARDIAN_PROMETHEUS_CA_BUNDLE_PATH", Value: "/ca/bundle"})) }) diff --git a/pkg/enterprise/installation.go b/pkg/enterprise/installation.go index 30f1c96341..57ad67476c 100644 --- a/pkg/enterprise/installation.go +++ b/pkg/enterprise/installation.go @@ -18,7 +18,6 @@ import ( "errors" "fmt" - operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/dns" "github.com/tigera/operator/pkg/extensions" @@ -27,50 +26,54 @@ import ( "github.com/tigera/operator/pkg/render/monitor" ) -func registerInstallation(s *extensions.Set) { - s.RegisterSetup(operatorv1.CalicoEnterprise, setup) -} - -// setup is the Calico Enterprise setup phase. It builds the base render context -// and then does the controller-side work the modifiers can't: validating config -// and creating/fetching the certificates that feed the trusted bundle. -func setup(in extensions.Inputs) (extensions.RenderContext, error) { - rc := extensions.BaseRenderContext(in) +// controllerExtension is the Calico Enterprise controller-side hook for the +// installation controller. +type controllerExtension struct{} +// Validate rejects installation config Calico Enterprise does not support. +func (controllerExtension) Validate(cc extensions.ControllerContext) error { // Reject the unsupported zero reporter port. The port value itself is derived // in the node modifier; only this validation lives here. - if in.FelixConfiguration.Spec.PrometheusReporterPort != nil && *in.FelixConfiguration.Spec.PrometheusReporterPort == 0 { - return rc, errors.New("felixConfiguration prometheusReporterPort=0 not supported") + if cc.FelixConfiguration.Spec.PrometheusReporterPort != nil && *cc.FelixConfiguration.Spec.PrometheusReporterPort == 0 { + return errors.New("felixConfiguration prometheusReporterPort=0 not supported") } + return nil +} + +// ExtendContext does the controller-side work the modifiers can't: creating and +// fetching the certificates that feed the trusted bundle, returning the render +// context with the produced node prometheus keypair layered on. +func (controllerExtension) ExtendContext(cc extensions.ControllerContext) (extensions.RenderContext, error) { + rc := cc.RenderContext - nodePrometheusTLS, err := in.CertificateManager.GetOrCreateKeyPair( - in.Client, + nodePrometheusTLS, err := cc.CertificateManager.GetOrCreateKeyPair( + cc.Client, render.NodePrometheusTLSServerSecret, common.OperatorNamespace(), - dns.GetServiceDNSNames(render.CalicoNodeMetricsService, common.CalicoNamespace, in.ClusterDomain), + dns.GetServiceDNSNames(render.CalicoNodeMetricsService, common.CalicoNamespace, cc.ClusterDomain), ) if err != nil { return rc, fmt.Errorf("error creating node prometheus TLS certificate: %w", err) } if nodePrometheusTLS != nil { - in.TrustedBundle.AddCertificates(nodePrometheusTLS) + cc.TrustedBundle.AddCertificates(nodePrometheusTLS) } rc.NodePrometheusTLS = nodePrometheusTLS - prometheusClientCert, err := in.CertificateManager.GetCertificate(in.Client, monitor.PrometheusClientTLSSecretName, common.OperatorNamespace()) + prometheusClientCert, err := cc.CertificateManager.GetCertificate(cc.Client, monitor.PrometheusClientTLSSecretName, common.OperatorNamespace()) if err != nil { return rc, fmt.Errorf("unable to fetch prometheus certificate: %w", err) } if prometheusClientCert != nil { - in.TrustedBundle.AddCertificates(prometheusClientCert) + cc.TrustedBundle.AddCertificates(prometheusClientCert) } - esgwCertificate, err := in.CertificateManager.GetCertificate(in.Client, relasticsearch.PublicCertSecret, common.OperatorNamespace()) + esgwCertificate, err := cc.CertificateManager.GetCertificate(cc.Client, relasticsearch.PublicCertSecret, common.OperatorNamespace()) if err != nil { return rc, fmt.Errorf("failed to retrieve / validate %s: %w", relasticsearch.PublicCertSecret, err) } if esgwCertificate != nil { - in.TrustedBundle.AddCertificates(esgwCertificate) + cc.TrustedBundle.AddCertificates(esgwCertificate) } return rc, nil diff --git a/pkg/enterprise/installation_test.go b/pkg/enterprise/installation_test.go index bc704a2429..f968306b96 100644 --- a/pkg/enterprise/installation_test.go +++ b/pkg/enterprise/installation_test.go @@ -31,32 +31,31 @@ import ( "github.com/tigera/operator/pkg/extensions" ) -var _ = Describe("installation setup", func() { +var _ = Describe("installation controller extension", func() { It("rejects a zero prometheus reporter port", func() { port := 0 - in := newInputs(operatorv1.CalicoEnterprise) - in.FelixConfiguration = &v3.FelixConfiguration{ + cc := newControllerContext(operatorv1.CalicoEnterprise) + cc.FelixConfiguration = &v3.FelixConfiguration{ Spec: v3.FelixConfigurationSpec{PrometheusReporterPort: &port}, } - _, err := ext.BuildContext(in) - Expect(err).To(HaveOccurred()) + Expect(ext.Validate(cc)).To(HaveOccurred()) }) It("creates the node prometheus keypair for the enterprise variant", func() { - rc, err := ext.BuildContext(newInputs(operatorv1.CalicoEnterprise)) + rc, err := ext.ExtendContext(newControllerContext(operatorv1.CalicoEnterprise)) Expect(err).NotTo(HaveOccurred()) Expect(rc.NodePrometheusTLS).NotTo(BeNil()) }) It("is a no-op for the Calico variant", func() { - rc, err := ext.BuildContext(newInputs(operatorv1.Calico)) + rc, err := ext.ExtendContext(newControllerContext(operatorv1.Calico)) Expect(err).NotTo(HaveOccurred()) Expect(rc.NodePrometheusTLS).To(BeNil()) }) }) -func newInputs(variant operatorv1.ProductVariant) extensions.Inputs { +func newControllerContext(variant operatorv1.ProductVariant) extensions.ControllerContext { scheme := runtime.NewScheme() Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) c := ctrlrfake.DefaultFakeClientBuilder(scheme).Build() @@ -65,13 +64,15 @@ func newInputs(variant operatorv1.ProductVariant) extensions.Inputs { Expect(err).NotTo(HaveOccurred()) trustedBundle := certManager.CreateTrustedBundle() - return extensions.Inputs{ + return extensions.ControllerContext{ + RenderContext: extensions.RenderContext{ + Installation: &operatorv1.InstallationSpec{Variant: variant}, + FelixConfiguration: &v3.FelixConfiguration{}, + TrustedBundle: trustedBundle, + ClusterDomain: "cluster.local", + }, Ctx: context.Background(), Client: c, - Installation: &operatorv1.InstallationSpec{Variant: variant}, - FelixConfiguration: &v3.FelixConfiguration{}, CertificateManager: certManager, - TrustedBundle: trustedBundle, - ClusterDomain: "cluster.local", } } diff --git a/pkg/enterprise/node.go b/pkg/enterprise/node.go index d0ca3e2135..60b02ccdff 100644 --- a/pkg/enterprise/node.go +++ b/pkg/enterprise/node.go @@ -46,20 +46,16 @@ const ( installCNIContainerName = "install-cni" ) -func registerNode(s *extensions.Set) { - s.Register(operatorv1.CalicoEnterprise, render.ComponentNameNode, extensions.ComponentExtension{ - Image: func(in *operatorv1.InstallationSpec) components.Component { - return components.ComponentTigeraNode - }, - Modify: modifyNode, +func registerNode(v *extensions.Variant) { + v.Image(render.ComponentNameNode, func(in *operatorv1.InstallationSpec) components.Component { + return components.ComponentTigeraNode }) + v.Modify(render.ComponentNameNode, modifyNode) // The node component renders the cni-plugins init container; its image // resolves through its own override key. - s.Register(operatorv1.CalicoEnterprise, render.ComponentNameCNIPlugins, extensions.ComponentExtension{ - Image: func(in *operatorv1.InstallationSpec) components.Component { - return components.ComponentTigeraCNIPlugins - }, + v.Image(render.ComponentNameCNIPlugins, func(in *operatorv1.InstallationSpec) components.Component { + return components.ComponentTigeraCNIPlugins }) } diff --git a/pkg/enterprise/register.go b/pkg/enterprise/register.go index d1f81451b1..e0d0532b31 100644 --- a/pkg/enterprise/register.go +++ b/pkg/enterprise/register.go @@ -14,19 +14,31 @@ package enterprise -import "github.com/tigera/operator/pkg/extensions" +import ( + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/extensions" +) -// New builds the extension Set for the in-repo Calico Enterprise variant: every -// component modifier, image override, and the installation setup. The operator -// is handed this Set at startup (the core operator is handed none). After the -// monorepo split this is what calico-private's main will construct instead. +// New builds the extension Set for the in-repo Calico Enterprise variant: the +// controller extension, every component modifier, and the image overrides. The +// operator is handed this Set at startup (the core operator is handed none). +// After the monorepo split this is what calico-private's main will construct +// instead. func New() *extensions.Set { s := extensions.NewSet() - registerTypha(s) - registerNode(s) - registerWindows(s) - registerGuardian(s) - registerInstallation(s) - registerAPIServer(s) + + ent := s.Variant(operatorv1.CalicoEnterprise) + ent.Controller(controllerExtension{}) + registerTypha(ent) + registerNode(ent) + registerWindows(ent) + registerGuardian(ent) + registerAPIServer(ent) + + // When the enterprise operator manages a Calico installation, clean up the + // Enterprise objects left behind by a prior Enterprise installation. + cal := s.Variant(operatorv1.Calico) + registerAPIServerCleanup(cal) + return s } diff --git a/pkg/enterprise/typha.go b/pkg/enterprise/typha.go index ce3a434bbe..77ce79000c 100644 --- a/pkg/enterprise/typha.go +++ b/pkg/enterprise/typha.go @@ -20,15 +20,12 @@ import ( rbacv1 "k8s.io/api/rbac/v1" "sigs.k8s.io/controller-runtime/pkg/client" - operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" ) -func registerTypha(s *extensions.Set) { - s.Register(operatorv1.CalicoEnterprise, render.ComponentNameTypha, extensions.ComponentExtension{ - Modify: modifyTypha, - }) +func registerTypha(v *extensions.Variant) { + v.Modify(render.ComponentNameTypha, modifyTypha) } func modifyTypha(ctx extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { diff --git a/pkg/enterprise/windows.go b/pkg/enterprise/windows.go index 7883429e41..2b2d43753a 100644 --- a/pkg/enterprise/windows.go +++ b/pkg/enterprise/windows.go @@ -17,7 +17,6 @@ package enterprise import ( "fmt" - "github.com/sirupsen/logrus" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -36,29 +35,21 @@ import ( // felix env and node volume mounts, so they receive the same enterprise layering. var windowsNodeContainers = map[string]bool{"felix": true, "node": true, "confd": true} -func registerWindows(s *extensions.Set) { - s.Register(operatorv1.CalicoEnterprise, render.ComponentNameWindowsNodeImg, extensions.ComponentExtension{ - Image: func(*operatorv1.InstallationSpec) components.Component { return components.ComponentTigeraNodeWindows }, +func registerWindows(v *extensions.Variant) { + v.Image(render.ComponentNameWindowsNodeImg, func(*operatorv1.InstallationSpec) components.Component { + return components.ComponentTigeraNodeWindows }) - s.Register(operatorv1.CalicoEnterprise, render.ComponentNameWindowsCNIImg, extensions.ComponentExtension{ - Image: func(*operatorv1.InstallationSpec) components.Component { return components.ComponentTigeraCNIWindows }, - }) - s.Register(operatorv1.CalicoEnterprise, render.ComponentNameWindows, extensions.ComponentExtension{ - Modify: modifyWindows, + v.Image(render.ComponentNameWindowsCNIImg, func(*operatorv1.InstallationSpec) components.Component { + return components.ComponentTigeraCNIWindows }) + extensions.RegisterModifier(v, render.ComponentNameWindows, modifyWindows) } // modifyWindows layers Calico Enterprise behavior onto the rendered // calico-node-windows objects: the node-metrics Service and the Enterprise // daemonset configuration (flow/DNS log env, prometheus reporter, trusted DNS // servers, the calico log volume, and the prometheus reporter keypair mount). -func modifyWindows(ctx extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { - wc, ok := ctx.Component.(render.WindowsExtensionContext) - if !ok { - logrus.Errorf("BUG: windows modifier got %T, want render.WindowsExtensionContext; leaving objects unchanged", ctx.Component) - return objs, del - } - +func modifyWindows(ctx extensions.RenderContext, wc render.WindowsExtensionContext, objs, del []client.Object) ([]client.Object, []client.Object) { if ds, ok := extensions.FindObject[*appsv1.DaemonSet](objs, common.WindowsDaemonSetName); ok { modifyWindowsDaemonSet(ctx, wc, ds) } diff --git a/pkg/enterprise/windows_test.go b/pkg/enterprise/windows_test.go index b6c6b0c2d0..d0c11c6642 100644 --- a/pkg/enterprise/windows_test.go +++ b/pkg/enterprise/windows_test.go @@ -85,19 +85,22 @@ var _ = Describe("windows enterprise modifier", func() { return nil } - ctxFor := func(provider operatorv1.Provider, tls certificatemanagement.KeyPairInterface, bundle certificatemanagement.TrustedBundleRO) extensions.RenderContext { + ctxFor := func(provider operatorv1.Provider) extensions.RenderContext { return extensions.RenderContext{ Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise, KubernetesProvider: provider}, - Component: render.WindowsExtensionContext{ - NodeReporterMetricsPort: 9081, - PrometheusServerTLS: tls, - TrustedBundle: bundle, - }, + } + } + + wcFor := func(tls certificatemanagement.KeyPairInterface, bundle certificatemanagement.TrustedBundleRO) render.WindowsExtensionContext { + return render.WindowsExtensionContext{ + NodeReporterMetricsPort: 9081, + PrometheusServerTLS: tls, + TrustedBundle: bundle, } } It("appends the node-metrics service", func() { - out, _ := applyExtensions(ext, render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone, nil, nil), newObjs(), nil) + out, _ := applyExtensionsWithContext(ext, render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone), wcFor(nil, nil), newObjs(), nil) svc, ok := extensions.FindObject[*corev1.Service](out, render.WindowsNodeMetricsService) Expect(ok).To(BeTrue()) Expect(svc.Namespace).To(Equal(common.CalicoNamespace)) @@ -105,7 +108,7 @@ var _ = Describe("windows enterprise modifier", func() { }) It("swaps the cni log mount for the calico log volume and adds enterprise env", func() { - out, _ := applyExtensions(ext, render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone, nil, nil), newObjs(), nil) + out, _ := applyExtensionsWithContext(ext, render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone), wcFor(nil, nil), newObjs(), nil) d := ds(out) Expect(d.Spec.Template.Spec.Volumes).To(ContainElement(HaveField("Name", "var-log-calico"))) @@ -122,7 +125,7 @@ var _ = Describe("windows enterprise modifier", func() { }) It("sets the trusted DNS server on openshift", func() { - out, _ := applyExtensions(ext, render.ComponentNameWindows, ctxFor(operatorv1.ProviderOpenShift, nil, nil), newObjs(), nil) + out, _ := applyExtensionsWithContext(ext, render.ComponentNameWindows, ctxFor(operatorv1.ProviderOpenShift), wcFor(nil, nil), newObjs(), nil) Expect(container(ds(out), "node").Env).To(ContainElement(corev1.EnvVar{Name: "FELIX_DNSTRUSTEDSERVERS", Value: "k8s-service:openshift-dns/dns-default"})) }) @@ -136,7 +139,7 @@ var _ = Describe("windows enterprise modifier", func() { Expect(err).NotTo(HaveOccurred()) bundle := cm.CreateTrustedBundle() - out, _ := applyExtensions(ext, render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone, tls, bundle), newObjs(), nil) + out, _ := applyExtensionsWithContext(ext, render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone), wcFor(tls, bundle), newObjs(), nil) d := ds(out) Expect(d.Spec.Template.Spec.Volumes).To(ContainElement(tls.Volume())) diff --git a/pkg/extensions/controllerextension.go b/pkg/extensions/controllerextension.go new file mode 100644 index 0000000000..ae22f5fd40 --- /dev/null +++ b/pkg/extensions/controllerextension.go @@ -0,0 +1,58 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extensions + +import ( + "context" + + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/tigera/operator/pkg/controller/certificatemanager" +) + +// ControllerExtension is a variant's controller-side reconcile hook. The +// installation controller calls it to do the work core can't: reject +// unsupported configuration (Validate) and create the controller-side artifacts +// - certificates, trusted bundle additions - that feed the render context +// (ExtendContext). A variant registers at most one; the core operator registers +// none and runs with the base behavior. +type ControllerExtension interface { + // Validate rejects configuration the variant does not support, before any + // rendering happens. + Validate(cc ControllerContext) error + + // ExtendContext does the controller-side work the render modifiers can't + // (creating certificates, extending the trusted bundle) and returns the + // RenderContext those modifiers read, or an error that aborts the reconcile. + ExtendContext(cc ControllerContext) (RenderContext, error) +} + +// ControllerContext is the controller-phase context, the corollary to the +// render-phase RenderContext. It is the embedded RenderContext (the same data +// the render phase sees) plus the controller-side machinery a ControllerExtension +// needs to produce artifacts: a client, a certificate manager, a context. Those +// deps live here, not on RenderContext, so the modifiers that read RenderContext +// can't do I/O - they only transform objects. +// +// The controller fills the embedded RenderContext's data fields and the deps; +// ExtendContext does its work, sets the produced artifacts (e.g. +// NodePrometheusTLS) on the embedded context, and returns it. +type ControllerContext struct { + RenderContext + + Ctx context.Context + Client client.Client + CertificateManager certificatemanager.CertificateManager +} diff --git a/pkg/extensions/controllerextension_test.go b/pkg/extensions/controllerextension_test.go new file mode 100644 index 0000000000..a0fc51275b --- /dev/null +++ b/pkg/extensions/controllerextension_test.go @@ -0,0 +1,104 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extensions_test + +import ( + "errors" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/extensions" +) + +var _ = Describe("controller extension", func() { + var s *extensions.Set + BeforeEach(func() { + s = extensions.NewSet() + }) + + It("returns the base render context when the variant has no extension", func() { + install := &operatorv1.InstallationSpec{Variant: operatorv1.Calico} + rc, err := s.ExtendContext(extensions.ControllerContext{ + RenderContext: extensions.RenderContext{Installation: install, ClusterDomain: "cluster.local"}, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(rc.Installation).To(BeIdenticalTo(install)) + Expect(rc.ClusterDomain).To(Equal("cluster.local")) + Expect(rc.NodePrometheusTLS).To(BeNil()) + }) + + It("runs the extension registered for the installation variant", func() { + s.Variant(operatorv1.CalicoEnterprise).Controller(fakeController{}) + rc, err := s.ExtendContext(enterpriseContext()) + Expect(err).NotTo(HaveOccurred()) + Expect(rc.ClusterDomain).To(Equal("from-fake")) + }) + + It("ignores an extension registered for a different variant", func() { + s.Variant(operatorv1.CalicoEnterprise).Controller(fakeController{}) + rc, err := s.ExtendContext(extensions.ControllerContext{ + RenderContext: extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}, ClusterDomain: "real"}, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(rc.ClusterDomain).To(Equal("real")) + }) + + It("surfaces the extension error", func() { + s.Variant(operatorv1.CalicoEnterprise).Controller(fakeController{err: errors.New("boom")}) + _, err := s.ExtendContext(enterpriseContext()) + Expect(err).To(MatchError("boom")) + }) + + It("runs the extension's validation", func() { + s.Variant(operatorv1.CalicoEnterprise).Controller(fakeController{validateErr: errors.New("invalid")}) + Expect(s.Validate(enterpriseContext())).To(MatchError("invalid")) + }) + + It("returns the base context and no validation error for a nil Set", func() { + var nilSet *extensions.Set + cc := enterpriseContext() + cc.ClusterDomain = "real" + rc, err := nilSet.ExtendContext(cc) + Expect(err).NotTo(HaveOccurred()) + Expect(rc.ClusterDomain).To(Equal("real")) + Expect(nilSet.Validate(cc)).NotTo(HaveOccurred()) + }) +}) + +func enterpriseContext() extensions.ControllerContext { + return extensions.ControllerContext{ + RenderContext: extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}}, + } +} + +// fakeController is a ControllerExtension whose Validate and ExtendContext return +// configurable results. +type fakeController struct { + err error + validateErr error +} + +func (f fakeController) Validate(_ extensions.ControllerContext) error { + return f.validateErr +} + +func (f fakeController) ExtendContext(_ extensions.ControllerContext) (extensions.RenderContext, error) { + if f.err != nil { + return extensions.RenderContext{}, f.err + } + return extensions.RenderContext{ClusterDomain: "from-fake"}, nil +} diff --git a/pkg/extensions/decorate_helpers_test.go b/pkg/extensions/decorate_helpers_test.go index 227c4f552a..86afb71df0 100644 --- a/pkg/extensions/decorate_helpers_test.go +++ b/pkg/extensions/decorate_helpers_test.go @@ -24,8 +24,8 @@ import ( // stubExtComponent adapts raw object lists to a render.Component so a registered // extension can be exercised through Set.Decorate, the same seam the component -// handler uses. key selects the extension; extCtx is delivered as the per-component -// ExtensionContext. +// handler uses. key selects the extension; extCtx is delivered as the component's +// ExtensionContext (the typed config a RegisterModifier modifier reads). type stubExtComponent struct { key string extCtx any @@ -57,9 +57,15 @@ func (s stubExtComponent) ExtensionContext() any { } // applyExtensions decorates a stub component holding the given objects with the -// extension registered under key in s, then renders it - returning the decorated -// create and delete lists. +// extension registered under key, then renders it. For a modifier that needs the +// component's typed config, use applyExtensionsWithContext. func applyExtensions(s *extensions.Set, key string, ctx extensions.RenderContext, create, del []client.Object) ([]client.Object, []client.Object) { - stub := stubExtComponent{key: key, extCtx: ctx.Component, create: create, delete: del} + return applyExtensionsWithContext(s, key, ctx, nil, create, del) +} + +// applyExtensionsWithContext is applyExtensions for a modifier that reads the +// component's typed config: extCtx is delivered as the stub's ExtensionContext. +func applyExtensionsWithContext(s *extensions.Set, key string, ctx extensions.RenderContext, extCtx any, create, del []client.Object) ([]client.Object, []client.Object) { + stub := stubExtComponent{key: key, extCtx: extCtx, create: create, delete: del} return s.Decorate(stub, ctx).Objects() } diff --git a/pkg/extensions/doc.go b/pkg/extensions/doc.go index a505f82fa1..ec06d9597b 100644 --- a/pkg/extensions/doc.go +++ b/pkg/extensions/doc.go @@ -16,23 +16,29 @@ // Enterprise) use to layer variant-specific behavior onto the core operator's // render output, so core code never branches on variant. // -// Everything keys off the installation Variant, and registration is per -// variant, so a registered hook only ever runs for its own variant and never -// re-checks it. There are two phases: +// A Set holds the extensions for every variant. Per reconcile the controller +// selects one Variant from the installation's variant, so a registered hook only +// ever runs for its own variant and never re-checks it. A Variant bundles two +// kinds of extension: // -// Setup is the controller-side phase. It runs once per reconcile in the -// installation controller, has cluster access (Client, CertificateManager), and -// does the side-effecting work a pure render hook can't: creating certificates, -// extending the trusted bundle, validating config. It returns the RenderContext -// - the read-only baton passed to the render phase. Register one per variant -// with RegisterSetup; the controller runs it with BuildContext. +// A ControllerExtension is the controller-side hook. It runs once per reconcile +// in the installation controller, has cluster access (Client, +// CertificateManager) via the ControllerContext, and does the side-effecting +// work a pure render hook can't: rejecting unsupported config (Validate) and +// creating certificates / extending the trusted bundle (ExtendContext). It +// returns the RenderContext, the read-only baton passed to the render phase. // -// Extension is the render phase: pure, per-component hooks that run after a -// component builds its objects. Its Image field overrides the component's image -// (resolved during ResolveImages), and its Modify field post-processes the -// rendered objects (run at the componentHandler). Register one per component -// with Register. +// Per-component modifiers are the render phase: pure hooks that run after a +// component builds its objects. An image override swaps the component's image +// (resolved during ResolveImages); a Modifier post-processes the rendered +// objects (run at the componentHandler, which renders the decorated component). +// Register a modifier with Variant.Modify, or with RegisterModifier when it +// needs the component's own typed config. // -// A variant wires up its setup and extensions in one place at startup - see -// pkg/enterprise. +// ControllerContext (controller phase) and RenderContext (render phase) are a +// pair: ControllerContext embeds RenderContext and adds the cluster-access deps, +// which is why modifiers, given only a RenderContext, can't do I/O. +// +// A variant wires up its controller extension and modifiers in one place at +// startup - see pkg/enterprise. package extensions diff --git a/pkg/extensions/extension.go b/pkg/extensions/extension.go index e9ef998a6b..3e00227624 100644 --- a/pkg/extensions/extension.go +++ b/pkg/extensions/extension.go @@ -16,79 +16,16 @@ package extensions import ( "sigs.k8s.io/controller-runtime/pkg/client" - - operatorv1 "github.com/tigera/operator/api/v1" - "github.com/tigera/operator/pkg/render" ) -// ComponentExtension is everything a variant layers onto one render component. -// Every field is optional: a component that only needs a different image sets -// Image and leaves Modify nil, and vice versa. This is the single registration a -// variant makes per component, so all of that component's variance lives in one -// place. -type ComponentExtension struct { - // Image overrides the component's image. Resolved during ResolveImages, in - // the render package, via the imageoverride leaf. - Image ImageOverride - - // Modify post-processes the component's rendered objects, after Objects(). - Modify Modifier -} - // Modifier post-processes the objects a render component produced. It receives // the component's create and delete lists and returns the (possibly extended) // lists. A modifier may mutate matched objects, append objects to create, and -// append objects to delete (e.g. to clean up resources another variant left -// behind). A modifier runs only for the variant it was registered under, so it -// need not re-check the variant. +// append objects to delete (e.g. to clean up resources a prior variant left +// behind). It runs only for the variant it is registered under, so it need not +// re-check the variant. type Modifier func(ctx RenderContext, create, delete []client.Object) (newCreate, newDelete []client.Object) -type modifierKey struct { - variant operatorv1.ProductVariant - component string -} - -// Decorate wraps component with the variant extension registered for it, so -// that when the handler renders the component its objects are post-processed by -// the modifier registered for the component and the installation's variant. A -// decorated component is itself a render.Component, so it flows through the -// component handler exactly like any other. Returns component unchanged when it -// exposes no extension point, when no modifier is registered for it, when no -// installation is set, or on a nil Set (the core operator registers no -// extensions). -func (s *Set) Decorate(component render.Component, ctx RenderContext) render.Component { - if s == nil || ctx.Installation == nil { - return component - } - ext, ok := component.(render.Extensible) - if !ok { - return component - } - modify, ok := s.modifiers[modifierKey{ctx.Installation.Variant, ext.ModifierKey()}] - if !ok { - return component - } - if p, ok := component.(render.ExtensionContextProvider); ok { - ctx.Component = p.ExtensionContext() - } - return &decoratedComponent{Component: component, ctx: ctx, modify: modify} -} - -// decoratedComponent is the render.Component produced by Decorate: it renders -// its embedded base component and then runs the variant modifier over the -// result. It embeds the base render.Component, so ResolveImages, SupportedOSType, -// and Ready delegate to the base; only Objects is augmented. -type decoratedComponent struct { - render.Component - ctx RenderContext - modify Modifier -} - -func (d *decoratedComponent) Objects() ([]client.Object, []client.Object) { - create, del := d.Component.Objects() - return d.modify(d.ctx, create, del) -} - // FindObject returns the first object of type T with the given name. func FindObject[T client.Object](objs []client.Object, name string) (T, bool) { var zero T diff --git a/pkg/extensions/extension_test.go b/pkg/extensions/extension_test.go index 8cbb0a7c89..0ea3359345 100644 --- a/pkg/extensions/extension_test.go +++ b/pkg/extensions/extension_test.go @@ -34,13 +34,11 @@ var _ = Describe("extension registry", func() { entCtx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}} It("applies a registered modifier to the matching component and variant", func() { - s.Register(operatorv1.CalicoEnterprise, "test", extensions.ComponentExtension{ - Modify: func(ctx extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { - cm, ok := extensions.FindObject[*corev1.ConfigMap](objs, "cm") - Expect(ok).To(BeTrue()) - cm.Data = map[string]string{"k": "v"} - return append(objs, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "extra"}}), del - }, + s.Variant(operatorv1.CalicoEnterprise).Modify("test", func(ctx extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { + cm, ok := extensions.FindObject[*corev1.ConfigMap](objs, "cm") + Expect(ok).To(BeTrue()) + cm.Data = map[string]string{"k": "v"} + return append(objs, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "extra"}}), del }) in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} @@ -53,10 +51,8 @@ var _ = Describe("extension registry", func() { }) It("lets a modifier append to the delete list", func() { - s.Register(operatorv1.CalicoEnterprise, "test", extensions.ComponentExtension{ - Modify: func(_ extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { - return objs, append(del, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "stale"}}) - }, + s.Variant(operatorv1.CalicoEnterprise).Modify("test", func(_ extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { + return objs, append(del, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "stale"}}) }) in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} @@ -73,10 +69,8 @@ var _ = Describe("extension registry", func() { }) It("does not apply a modifier registered for a different variant", func() { - s.Register(operatorv1.CalicoEnterprise, "test", extensions.ComponentExtension{ - Modify: func(_ extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { - return append(objs, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "extra"}}), del - }, + s.Variant(operatorv1.CalicoEnterprise).Modify("test", func(_ extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { + return append(objs, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "extra"}}), del }) calicoCtx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}} @@ -91,16 +85,14 @@ var _ = Describe("extension registry", func() { Expect(out).To(Equal(in)) }) - It("replaces rather than stacks when a (variant, component) is registered twice", func() { - add := func(name string) extensions.ComponentExtension { - return extensions.ComponentExtension{ - Modify: func(_ extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { - return append(objs, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: name}}), del - }, - } + It("replaces rather than stacks when a component modifier is registered twice", func() { + add := func(name string) { + s.Variant(operatorv1.CalicoEnterprise).Modify("test", func(_ extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { + return append(objs, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: name}}), del + }) } - s.Register(operatorv1.CalicoEnterprise, "test", add("first")) - s.Register(operatorv1.CalicoEnterprise, "test", add("second")) + add("first") + add("second") out, _ := applyExtensions(s, "test", entCtx, nil, nil) Expect(out).To(HaveLen(1)) diff --git a/pkg/extensions/image_test.go b/pkg/extensions/image_test.go index dcc4876fdd..03d4abbce2 100644 --- a/pkg/extensions/image_test.go +++ b/pkg/extensions/image_test.go @@ -27,10 +27,8 @@ var _ = Describe("image overrides", func() { var s *extensions.Set BeforeEach(func() { s = extensions.NewSet() - s.Register(operatorv1.CalicoEnterprise, "node", extensions.ComponentExtension{ - Image: func(in *operatorv1.InstallationSpec) components.Component { - return components.ComponentTigeraNode - }, + s.Variant(operatorv1.CalicoEnterprise).Image("node", func(in *operatorv1.InstallationSpec) components.Component { + return components.ComponentTigeraNode }) }) diff --git a/pkg/extensions/rendercontext.go b/pkg/extensions/rendercontext.go index c03389c153..95daec43aa 100644 --- a/pkg/extensions/rendercontext.go +++ b/pkg/extensions/rendercontext.go @@ -22,14 +22,15 @@ import ( // RenderContext carries reconcile-derived inputs from controllers into render // modifiers. Core operator code never reads these fields - only registered -// modifiers do. -// Three kinds of value live here: +// modifiers do. Two kinds of value live here: // - raw cluster state gathered generically (Installation, FelixConfiguration, -// ClusterDomain) that modifiers derive their own values from, +// ClusterDomain) that modifiers derive their own values from, and // - controller-produced artifacts (TrustedBundle, NodePrometheusTLS) that can -// only be created controller-side because they have cluster side effects, and -// - Component, the per-component context the component being modified supplies -// for config a modifier can't derive from the fields above. +// only be created controller-side because they have cluster side effects. +// +// Per-component config a modifier needs but can't derive from these fields is +// not carried here; it flows to the modifier as a typed argument (see +// RegisterModifier), supplied by the component via render.ExtensionContextProvider. type RenderContext struct { Installation *operatorv1.InstallationSpec FelixConfiguration *v3.FelixConfiguration @@ -38,17 +39,9 @@ type RenderContext struct { // TrustedBundle is the shared CA bundle for the calico-system namespace. TrustedBundle certificatemanagement.TrustedBundle - // NodePrometheusTLS is created by the enterprise setup (it has cluster side - // effects, so it can't be built in a modifier). The node modifier is its only - // consumer: it mounts the keypair onto the daemonset and sets the + // NodePrometheusTLS is created by the enterprise controller extension (it has + // cluster side effects, so it can't be built in a modifier). The node modifier + // is its only consumer: it mounts the keypair onto the daemonset and sets the // FELIX_PROMETHEUSREPORTER* certificate env vars. NodePrometheusTLS certificatemanagement.KeyPairInterface - - // Component is per-component context that the component being modified supplies - // via render.ExtensionContextProvider - config a modifier needs but can't - // derive from the fields above (e.g. a keypair the component's own controller - // created, or a CR field only that controller reads). The componentHandler - // sets it per component before applying the modifier; a modifier type-asserts - // it to the component's own context type. Nil when the component supplies none. - Component any } diff --git a/pkg/extensions/set.go b/pkg/extensions/set.go index 08aa52f259..42a76267ad 100644 --- a/pkg/extensions/set.go +++ b/pkg/extensions/set.go @@ -18,56 +18,89 @@ import ( operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/imageoverride" + "github.com/tigera/operator/pkg/render" ) -// Set is the collection of variant extensions the operator runs with: the -// per-variant setups, the per-component modifiers, and the image overrides. The -// core operator runs with a nil/empty Set; an extension build (Calico -// Enterprise) constructs a populated one and hands it in through +// Set is all the variant extensions the operator runs with, indexed by product +// variant. The core operator runs with a nil Set; an extension build (e.g. +// Calico Enterprise) constructs a populated one and hands it in through // options.ControllerOptions. This replaces what used to be package-level // registries, so nothing is wired by import side effect. // -// The zero value is not usable; build one with NewSet. The methods that read it -// (BuildContext, Decorate, ResolveImage, Images) are nil-safe so the core -// operator can pass a nil Set and get base behavior. +// Per reconcile the controller selects one Variant from the installation's +// variant. The methods the controller calls (Decorate, Validate, ExtendContext, +// Images, ResolveImage) are nil-safe, so the core operator's nil Set yields base +// behavior. type Set struct { - setups map[operatorv1.ProductVariant]Setup - modifiers map[modifierKey]Modifier - images *imageoverride.Overrides + variants map[operatorv1.ProductVariant]*Variant + images *imageoverride.Overrides } -// NewSet returns an empty Set ready to register extensions into. +// NewSet returns an empty Set ready to register variant extensions into. func NewSet() *Set { return &Set{ - setups: map[operatorv1.ProductVariant]Setup{}, - modifiers: map[modifierKey]Modifier{}, - images: imageoverride.New(), + variants: map[operatorv1.ProductVariant]*Variant{}, + images: imageoverride.New(), } } -// Register installs e as the extension for the named component under the given -// variant. A (variant, component) pair has at most one extension; registration -// replaces any prior one. The image override and the modifier are stored -// separately, so a component can set either field or both. -func (s *Set) Register(variant operatorv1.ProductVariant, component string, e ComponentExtension) { - if e.Image != nil { - s.images.Register(variant, component, e.Image) +// Variant returns the extension bundle for v, creating an empty one if needed. +// Used at registration time to build up a variant's extensions. +func (s *Set) Variant(v operatorv1.ProductVariant) *Variant { + if s.variants[v] == nil { + s.variants[v] = &Variant{ + variant: v, + modifiers: map[string]decorator{}, + images: s.images, + } } - if e.Modify != nil { - s.modifiers[modifierKey{variant, component}] = e.Modify + return s.variants[v] +} + +// variant looks up the bundle for v, returning nil when none is registered. +// Nil-safe. +func (s *Set) variant(v operatorv1.ProductVariant) *Variant { + if s == nil { + return nil + } + return s.variants[v] +} + +// Decorate wraps component with the extension registered for it under the +// installation's variant, so that when the handler renders the component its +// objects are post-processed by that modifier. A decorated component is itself a +// render.Component, so it flows through the component handler like any other. +// Returns component unchanged when no extension applies. Nil-safe. +func (s *Set) Decorate(component render.Component, ctx RenderContext) render.Component { + if ctx.Installation == nil { + return component + } + return s.variant(ctx.Installation.Variant).decorate(component, ctx) +} + +// Validate runs the controller extension's validation for the installation's +// variant, or returns nil when no extension is registered. Nil-safe. +func (s *Set) Validate(cc ControllerContext) error { + if cc.Installation == nil { + return nil } + return s.variant(cc.Installation.Variant).validate(cc) } -// RegisterSetup installs setup as the controller-side setup phase for the given -// variant. Registration replaces any prior setup for that variant. -func (s *Set) RegisterSetup(variant operatorv1.ProductVariant, setup Setup) { - s.setups[variant] = setup +// ExtendContext runs the controller extension for the installation's variant and +// returns the resulting RenderContext, or the base render context when no +// extension is registered. Nil-safe. +func (s *Set) ExtendContext(cc ControllerContext) (RenderContext, error) { + if cc.Installation == nil { + return cc.RenderContext, nil + } + return s.variant(cc.Installation.Variant).extendContext(cc) } -// Images returns the image overrides. The render package resolves a component's -// image through these directly (the imageoverride leaf, so render need not -// import extensions). Safe to call on a nil Set, which returns nil overrides -// that resolve to the default image. +// Images returns the shared image override table. The render package resolves a +// component's image through it directly (the imageoverride leaf, so render need +// not import extensions). Nil-safe, returning nil overrides that resolve to the +// default image. func (s *Set) Images() *imageoverride.Overrides { if s == nil { return nil @@ -76,7 +109,7 @@ func (s *Set) Images() *imageoverride.Overrides { } // ResolveImage resolves key for the installation through the image overrides, -// returning def when no override applies. Safe to call on a nil Set. +// returning def when no override applies. Nil-safe. func (s *Set) ResolveImage(key string, def components.Component, in *operatorv1.InstallationSpec) components.Component { return s.Images().Resolve(key, def, in) } diff --git a/pkg/extensions/setup.go b/pkg/extensions/setup.go deleted file mode 100644 index 2eb0ef1b6b..0000000000 --- a/pkg/extensions/setup.go +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright (c) 2026 Tigera, Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package extensions - -import ( - "context" - - v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" - "sigs.k8s.io/controller-runtime/pkg/client" - - operatorv1 "github.com/tigera/operator/api/v1" - "github.com/tigera/operator/pkg/controller/certificatemanager" - "github.com/tigera/operator/pkg/tls/certificatemanagement" -) - -// Inputs is the reconcile state a Setup builds a RenderContext from. The -// installation controller populates it directly. It carries both the values -// that flow straight into the RenderContext and the side-effecting dependencies -// (Client, CertificateManager) a setup needs to produce controller-side -// artifacts. -type Inputs struct { - Ctx context.Context - Client client.Client - Installation *operatorv1.InstallationSpec - FelixConfiguration *v3.FelixConfiguration - CertificateManager certificatemanager.CertificateManager - TrustedBundle certificatemanagement.TrustedBundle - ClusterDomain string -} - -// Setup is a variant's controller-side reconcile phase. It performs the work -// modifiers can't (creating certificates, extending the trusted bundle, -// validating config) and returns the RenderContext that is then handed to that -// variant's modifiers - or an error that aborts the reconcile. -// -// This is the generic seam controllers use to extend base operator behavior; -// its first consumer is Calico Enterprise, but nothing here is enterprise -// specific. A setup runs only for the variant it was registered under, so it -// need not re-check the variant. -type Setup func(in Inputs) (RenderContext, error) - -// BaseRenderContext maps the generically-gathered inputs onto a RenderContext. -// Every setup builds on it, so the base fields are assembled in exactly one -// place. A setup layers its side-effect artifacts (e.g. NodePrometheusTLS) on -// top of the returned value. -func BaseRenderContext(in Inputs) RenderContext { - return RenderContext{ - Installation: in.Installation, - FelixConfiguration: in.FelixConfiguration, - ClusterDomain: in.ClusterDomain, - TrustedBundle: in.TrustedBundle, - } -} - -// BuildContext runs the setup registered for the installation variant and -// returns its RenderContext, or the base render context when the variant has no -// setup. Safe to call on a nil Set, which always returns the base context - the -// core operator registers no setups. -func (s *Set) BuildContext(in Inputs) (RenderContext, error) { - if s != nil && in.Installation != nil { - if setup, ok := s.setups[in.Installation.Variant]; ok { - return setup(in) - } - } - return BaseRenderContext(in), nil -} diff --git a/pkg/extensions/setup_test.go b/pkg/extensions/setup_test.go deleted file mode 100644 index 9ee58a6648..0000000000 --- a/pkg/extensions/setup_test.go +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright (c) 2026 Tigera, Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package extensions_test - -import ( - "errors" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - operatorv1 "github.com/tigera/operator/api/v1" - "github.com/tigera/operator/pkg/extensions" -) - -var _ = Describe("variant setup", func() { - var s *extensions.Set - BeforeEach(func() { - s = extensions.NewSet() - }) - - It("returns the base render context when the variant has no setup", func() { - install := &operatorv1.InstallationSpec{Variant: operatorv1.Calico} - rc, err := s.BuildContext(extensions.Inputs{ - Installation: install, - ClusterDomain: "cluster.local", - }) - Expect(err).NotTo(HaveOccurred()) - Expect(rc.Installation).To(BeIdenticalTo(install)) - Expect(rc.ClusterDomain).To(Equal("cluster.local")) - Expect(rc.NodePrometheusTLS).To(BeNil()) - }) - - It("uses the setup registered for the installation variant", func() { - s.RegisterSetup(operatorv1.CalicoEnterprise, fakeSetup(nil)) - rc, err := s.BuildContext(enterpriseInputs()) - Expect(err).NotTo(HaveOccurred()) - Expect(rc.ClusterDomain).To(Equal("from-fake")) - }) - - It("ignores a setup registered for a different variant", func() { - s.RegisterSetup(operatorv1.CalicoEnterprise, fakeSetup(nil)) - rc, err := s.BuildContext(extensions.Inputs{ - Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}, - ClusterDomain: "real", - }) - Expect(err).NotTo(HaveOccurred()) - Expect(rc.ClusterDomain).To(Equal("real")) - }) - - It("surfaces the setup error", func() { - s.RegisterSetup(operatorv1.CalicoEnterprise, fakeSetup(errors.New("boom"))) - _, err := s.BuildContext(enterpriseInputs()) - Expect(err).To(MatchError("boom")) - }) - - It("returns the base context for a nil Set", func() { - var nilSet *extensions.Set - in := enterpriseInputs() - in.ClusterDomain = "real" - rc, err := nilSet.BuildContext(in) - Expect(err).NotTo(HaveOccurred()) - Expect(rc.ClusterDomain).To(Equal("real")) - }) -}) - -func enterpriseInputs() extensions.Inputs { - return extensions.Inputs{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}} -} - -func fakeSetup(err error) extensions.Setup { - return func(_ extensions.Inputs) (extensions.RenderContext, error) { - if err != nil { - return extensions.RenderContext{}, err - } - return extensions.RenderContext{ClusterDomain: "from-fake"}, nil - } -} diff --git a/pkg/extensions/variant.go b/pkg/extensions/variant.go new file mode 100644 index 0000000000..175463d4e8 --- /dev/null +++ b/pkg/extensions/variant.go @@ -0,0 +1,138 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extensions + +import ( + "github.com/sirupsen/logrus" + "sigs.k8s.io/controller-runtime/pkg/client" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/imageoverride" + "github.com/tigera/operator/pkg/render" +) + +// Variant bundles everything that extends the core operator for one product +// variant: the controller-side hook, the per-component modifiers, and the image +// overrides. The Set selects one Variant per reconcile from the installation's +// variant, so within a Variant there is at most one extension per component and +// nothing here is itself keyed by variant. +type Variant struct { + variant operatorv1.ProductVariant + controller ControllerExtension + modifiers map[string]decorator + images *imageoverride.Overrides // shared with the owning Set +} + +// decorator wraps a base component, returning one whose Objects() are augmented +// by a registered modifier. +type decorator func(base render.Component, ctx RenderContext) render.Component + +// Controller registers the variant's controller-side extension. A variant has +// at most one; registering replaces any prior one. +func (v *Variant) Controller(c ControllerExtension) { + v.controller = c +} + +// Image registers an image override for the named component. +func (v *Variant) Image(component string, fn ImageOverride) { + v.images.Register(v.variant, component, fn) +} + +// Modify registers a modifier for a component that needs no per-component +// config. For components whose modifier needs the component's own typed config, +// use RegisterModifier. +func (v *Variant) Modify(component string, m Modifier) { + v.modifiers[component] = func(base render.Component, ctx RenderContext) render.Component { + return &decoratedComponent{Component: base, ctx: ctx, modify: m} + } +} + +// RegisterModifier registers a modifier for component whose modifier needs the +// component's own typed config. The component supplies it via +// render.ExtensionContextProvider; RegisterModifier asserts it to Cfg once, here, +// and hands the typed value to modify - so the modifier body needs no assertion. +// It is a free function because Go has no generic methods. +func RegisterModifier[Cfg any](v *Variant, component string, + modify func(ctx RenderContext, cfg Cfg, create, delete []client.Object) ([]client.Object, []client.Object), +) { + v.modifiers[component] = func(base render.Component, ctx RenderContext) render.Component { + provider, ok := base.(render.ExtensionContextProvider) + if !ok { + logrus.Errorf("BUG: component %q has a registered modifier but provides no extension context; leaving it unmodified", component) + return base + } + cfg, ok := provider.ExtensionContext().(Cfg) + if !ok { + var want Cfg + logrus.Errorf("BUG: component %q extension context is %T, want %T; leaving it unmodified", component, provider.ExtensionContext(), want) + return base + } + bound := func(ctx RenderContext, create, delete []client.Object) ([]client.Object, []client.Object) { + return modify(ctx, cfg, create, delete) + } + return &decoratedComponent{Component: base, ctx: ctx, modify: bound} + } +} + +// decorate wraps component with the modifier registered for its extension key, +// or returns it unchanged when the component exposes no extension point or none +// is registered. Nil-safe. +func (v *Variant) decorate(component render.Component, ctx RenderContext) render.Component { + if v == nil { + return component + } + ext, ok := component.(render.Extensible) + if !ok { + return component + } + build, ok := v.modifiers[ext.ModifierKey()] + if !ok { + return component + } + return build(component, ctx) +} + +// validate runs the controller extension's validation, or nil when the variant +// has none. Nil-safe. +func (v *Variant) validate(cc ControllerContext) error { + if v == nil || v.controller == nil { + return nil + } + return v.controller.Validate(cc) +} + +// extendContext runs the controller extension, or returns the base render +// context when the variant has none. Nil-safe. +func (v *Variant) extendContext(cc ControllerContext) (RenderContext, error) { + if v == nil || v.controller == nil { + return cc.RenderContext, nil + } + return v.controller.ExtendContext(cc) +} + +// decoratedComponent is the render.Component produced by decorate: it renders +// its embedded base component and then runs the variant modifier over the +// result. It embeds the base render.Component, so ResolveImages, SupportedOSType, +// and Ready delegate to the base; only Objects is augmented. +type decoratedComponent struct { + render.Component + ctx RenderContext + modify Modifier +} + +func (d *decoratedComponent) Objects() ([]client.Object, []client.Object) { + create, del := d.Component.Objects() + return d.modify(d.ctx, create, del) +} diff --git a/pkg/render/apiserver_test.go b/pkg/render/apiserver_test.go index 31b15779ee..57eeb83e66 100644 --- a/pkg/render/apiserver_test.go +++ b/pkg/render/apiserver_test.go @@ -68,12 +68,13 @@ import ( func apiServerObjects(c render.Component) ([]client.Object, []client.Object) { create, del := c.Objects() rc := extensions.RenderContext{} + var extCtx any if p, ok := c.(render.ExtensionContextProvider); ok { ec := p.ExtensionContext().(render.APIServerExtensionContext) rc.Installation = ec.Config.Installation - rc.Component = ec + extCtx = ec } - return applyExtensions(ext, render.ComponentNameAPIServer, rc, create, del) + return applyExtensionsWithContext(ext, render.ComponentNameAPIServer, rc, extCtx, create, del) } var _ = Describe("API server rendering tests (Calico Enterprise)", func() { diff --git a/pkg/render/decorate_helpers_test.go b/pkg/render/decorate_helpers_test.go index 750abb8df4..c748f56295 100644 --- a/pkg/render/decorate_helpers_test.go +++ b/pkg/render/decorate_helpers_test.go @@ -24,8 +24,8 @@ import ( // stubExtComponent adapts raw object lists to a render.Component so a registered // extension can be exercised through Set.Decorate, the same seam the component -// handler uses. key selects the extension; extCtx is delivered as the per-component -// ExtensionContext. +// handler uses. key selects the extension; extCtx is delivered as the component's +// ExtensionContext (the typed config a RegisterModifier modifier reads). type stubExtComponent struct { key string extCtx any @@ -57,9 +57,15 @@ func (s stubExtComponent) ExtensionContext() any { } // applyExtensions decorates a stub component holding the given objects with the -// extension registered under key in s, then renders it - returning the decorated -// create and delete lists. +// extension registered under key, then renders it. For a modifier that needs the +// component's typed config, use applyExtensionsWithContext. func applyExtensions(s *extensions.Set, key string, ctx extensions.RenderContext, create, del []client.Object) ([]client.Object, []client.Object) { - stub := stubExtComponent{key: key, extCtx: ctx.Component, create: create, delete: del} + return applyExtensionsWithContext(s, key, ctx, nil, create, del) +} + +// applyExtensionsWithContext is applyExtensions for a modifier that reads the +// component's typed config: extCtx is delivered as the stub's ExtensionContext. +func applyExtensionsWithContext(s *extensions.Set, key string, ctx extensions.RenderContext, extCtx any, create, del []client.Object) ([]client.Object, []client.Object) { + stub := stubExtComponent{key: key, extCtx: extCtx, create: create, delete: del} return s.Decorate(stub, ctx).Objects() } diff --git a/pkg/render/guardian_test.go b/pkg/render/guardian_test.go index 44e4e2fe4a..16f7653031 100644 --- a/pkg/render/guardian_test.go +++ b/pkg/render/guardian_test.go @@ -51,10 +51,11 @@ func guardianObjects(cfg *render.GuardianConfiguration) []client.Object { ExpectWithOffset(1, g.ResolveImages(nil)).To(BeNil()) objs, _ := g.Objects() rc := extensions.RenderContext{Installation: cfg.Installation} + var extCtx any if p, ok := g.(render.ExtensionContextProvider); ok { - rc.Component = p.ExtensionContext() + extCtx = p.ExtensionContext() } - out, _ := applyExtensions(ext, render.GuardianName, rc, objs, nil) + out, _ := applyExtensionsWithContext(ext, render.GuardianName, rc, extCtx, objs, nil) return out } @@ -113,10 +114,11 @@ var _ = Describe("Rendering tests", func() { // Apply the registered enterprise modifier the way the componentHandler // does, so these enterprise tests exercise the integrated output. rc := extensions.RenderContext{Installation: cfg.Installation} + var extCtx any if p, ok := g.(render.ExtensionContextProvider); ok { - rc.Component = p.ExtensionContext() + extCtx = p.ExtensionContext() } - resources, _ = applyExtensions(ext, render.GuardianName, rc, resources, nil) + resources, _ = applyExtensionsWithContext(ext, render.GuardianName, rc, extCtx, resources, nil) } BeforeEach(func() { @@ -349,10 +351,11 @@ var _ = Describe("Rendering tests", func() { // does, so the enterprise policy is exercised. For the Calico variant the // modifier is a no-op and the OSS policy is returned. rc := extensions.RenderContext{Installation: cfg.Installation} + var extCtx any if p, ok := g.(render.ExtensionContextProvider); ok { - rc.Component = p.ExtensionContext() + extCtx = p.ExtensionContext() } - resources, _ = applyExtensions(ext, render.ComponentNameGuardianPolicy, rc, objs, nil) + resources, _ = applyExtensionsWithContext(ext, render.ComponentNameGuardianPolicy, rc, extCtx, objs, nil) } Context("policy rendering based on variant and IncludeEgressNetworkPolicy", func() { diff --git a/pkg/render/windows_test.go b/pkg/render/windows_test.go index f15e185eda..903fb3349b 100644 --- a/pkg/render/windows_test.go +++ b/pkg/render/windows_test.go @@ -51,10 +51,11 @@ func renderWindows(cfg *render.WindowsConfiguration) []client.Object { ExpectWithOffset(1, comp.ResolveImages(nil)).To(BeNil()) objs, _ := comp.Objects() rc := extensions.RenderContext{Installation: cfg.Installation} + var extCtx any if p, ok := comp.(render.ExtensionContextProvider); ok { - rc.Component = p.ExtensionContext() + extCtx = p.ExtensionContext() } - out, _ := applyExtensions(ext, render.ComponentNameWindows, rc, objs, nil) + out, _ := applyExtensionsWithContext(ext, render.ComponentNameWindows, rc, extCtx, objs, nil) return out } From 9ae30122ff167c5f47e7abd99d98c15db1e26df0 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Thu, 18 Jun 2026 11:21:26 -0700 Subject: [PATCH 30/38] Clean up the extensions API per review Image overrides are plain components.Component values instead of funcs - an override only picks which image (registry, path, and FIPS handling are applied downstream in render), so registration reads v.Image(name, image) and the ImageOverride alias is gone. Rename the modifier RenderContext param from ctx to rc, split the RegisterModifier signature one arg per line, use slices.Contains over the local helpers, and trim the over-comments in the component handler and the ControllerExtension docs. --- .../installation/core_controller.go | 2 +- pkg/controller/utils/component.go | 7 -- pkg/enterprise/apiserver.go | 16 ++--- pkg/enterprise/decorate_helpers_test.go | 8 +-- pkg/enterprise/guardian.go | 4 +- pkg/enterprise/node.go | 64 ++++++++----------- pkg/enterprise/typha.go | 4 +- pkg/enterprise/windows.go | 20 +++--- pkg/extensions/controllerextension.go | 18 +++--- pkg/extensions/decorate_helpers_test.go | 8 +-- pkg/extensions/extension.go | 2 +- pkg/extensions/image.go | 24 ------- pkg/extensions/image_test.go | 4 +- pkg/extensions/variant.go | 33 +++++----- pkg/imageoverride/imageoverride.go | 19 +++--- pkg/render/decorate_helpers_test.go | 8 +-- 16 files changed, 92 insertions(+), 149 deletions(-) delete mode 100644 pkg/extensions/image.go diff --git a/pkg/controller/installation/core_controller.go b/pkg/controller/installation/core_controller.go index baeb11364e..47fdea89c1 100644 --- a/pkg/controller/installation/core_controller.go +++ b/pkg/controller/installation/core_controller.go @@ -1221,7 +1221,7 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile CertificateManager: certificateManager, } if err := r.opts.Extensions.Validate(cc); err != nil { - r.status.SetDegraded(operatorv1.ResourceValidationError, "Invalid installation configuration for variant", err, reqLogger) + r.status.SetDegraded(operatorv1.ResourceValidationError, "Invalid installation configuration", err, reqLogger) return reconcile.Result{}, err } renderCtx, err := r.opts.Extensions.ExtendContext(cc) diff --git a/pkg/controller/utils/component.go b/pkg/controller/utils/component.go index f805aef659..52eb2dc374 100644 --- a/pkg/controller/utils/component.go +++ b/pkg/controller/utils/component.go @@ -458,14 +458,7 @@ func resetMetadataForCreate(obj client.Object) { } func (c *componentHandler) CreateOrUpdateOrDelete(ctx context.Context, component render.Component, status status.StatusManager) error { - // Decorate the component with any registered variant extension before doing - // anything with it, so Ready, SupportedOSType, and Objects all reflect the - // extended component. Decorate is a no-op for components with no registered - // extension. if ext, ok := component.(render.Extensible); ok && c.extensions == nil { - // The component can be extended but this handler was built without an - // extension Set, so any registered extension silently won't run. That is - // a wiring bug in the controller, not a normal state. c.log.Info("BUG: extensible component rendered by a handler with no extension Set; extensions will not be applied", "component", ext.ModifierKey()) } component = c.extensions.Decorate(component, c.renderCtx) diff --git a/pkg/enterprise/apiserver.go b/pkg/enterprise/apiserver.go index 7a265877d1..b20f3579e5 100644 --- a/pkg/enterprise/apiserver.go +++ b/pkg/enterprise/apiserver.go @@ -16,6 +16,7 @@ package enterprise import ( "fmt" + "slices" "strings" appsv1 "k8s.io/api/apps/v1" @@ -64,7 +65,7 @@ func registerAPIServerCleanup(v *extensions.Variant) { // modifyAPIServer layers Calico Enterprise behavior onto the rendered API server objects: // the query server container and its volumes, audit logging on the aggregation API server // container, the Enterprise RBAC objects, and the query server port on the Service. -func modifyAPIServer(ctx extensions.RenderContext, ec render.APIServerExtensionContext, create, del []client.Object) ([]client.Object, []client.Object) { +func modifyAPIServer(rc extensions.RenderContext, ec render.APIServerExtensionContext, create, del []client.Object) ([]client.Object, []client.Object) { c := &apiServer{cfg: ec.Config, calicoImage: ec.CalicoImage} if dep, ok := extensions.FindObject[*appsv1.Deployment](create, render.APIServerName); ok { @@ -76,7 +77,7 @@ func modifyAPIServer(ctx extensions.RenderContext, ec render.APIServerExtensionC // Enterprise serves staged policies through the tiered-policy passthrough role. if role, ok := extensions.FindObject[*rbacv1.ClusterRole](create, "calico-tiered-policy-passthrough"); ok { for i := range role.Rules { - if contains(role.Rules[i].Resources, "networkpolicies") { + if slices.Contains(role.Rules[i].Resources, "networkpolicies") { role.Rules[i].Resources = append(role.Rules[i].Resources, "stagednetworkpolicies", "stagedglobalnetworkpolicies") } } @@ -143,7 +144,7 @@ func modifyAPIServer(ctx extensions.RenderContext, ec render.APIServerExtensionC // cleanupAPIServer deletes the Enterprise API server objects when running Calico, so a // cluster switched from Enterprise to Calico does not leave them behind. -func cleanupAPIServer(ctx extensions.RenderContext, ec render.APIServerExtensionContext, create, del []client.Object) ([]client.Object, []client.Object) { +func cleanupAPIServer(rc extensions.RenderContext, ec render.APIServerExtensionContext, create, del []client.Object) ([]client.Object, []client.Object) { c := &apiServer{cfg: ec.Config} del = append(del, c.tigeraAPIServerClusterRole(), c.tigeraAPIServerClusterRoleBinding()) @@ -235,15 +236,6 @@ func (c *apiServer) addQueryServerPort(s *corev1.Service) { }) } -func contains(s []string, v string) bool { - for _, x := range s { - if x == v { - return true - } - } - return false -} - func (c *apiServer) multiTenantSecretsRBAC() []client.Object { return render.TunnelSecretRBAC(render.APIServerSecretsRBACName, render.APIServerServiceAccountName, c.cfg.ManagementCluster, true) } diff --git a/pkg/enterprise/decorate_helpers_test.go b/pkg/enterprise/decorate_helpers_test.go index 184a801a71..bf9c6a72d7 100644 --- a/pkg/enterprise/decorate_helpers_test.go +++ b/pkg/enterprise/decorate_helpers_test.go @@ -59,13 +59,13 @@ func (s stubExtComponent) ExtensionContext() any { // applyExtensions decorates a stub component holding the given objects with the // extension registered under key, then renders it. For a modifier that needs the // component's typed config, use applyExtensionsWithContext. -func applyExtensions(s *extensions.Set, key string, ctx extensions.RenderContext, create, del []client.Object) ([]client.Object, []client.Object) { - return applyExtensionsWithContext(s, key, ctx, nil, create, del) +func applyExtensions(s *extensions.Set, key string, rc extensions.RenderContext, create, del []client.Object) ([]client.Object, []client.Object) { + return applyExtensionsWithContext(s, key, rc, nil, create, del) } // applyExtensionsWithContext is applyExtensions for a modifier that reads the // component's typed config: extCtx is delivered as the stub's ExtensionContext. -func applyExtensionsWithContext(s *extensions.Set, key string, ctx extensions.RenderContext, extCtx any, create, del []client.Object) ([]client.Object, []client.Object) { +func applyExtensionsWithContext(s *extensions.Set, key string, rc extensions.RenderContext, extCtx any, create, del []client.Object) ([]client.Object, []client.Object) { stub := stubExtComponent{key: key, extCtx: extCtx, create: create, delete: del} - return s.Decorate(stub, ctx).Objects() + return s.Decorate(stub, rc).Objects() } diff --git a/pkg/enterprise/guardian.go b/pkg/enterprise/guardian.go index 11895b0b29..4886889a29 100644 --- a/pkg/enterprise/guardian.go +++ b/pkg/enterprise/guardian.go @@ -46,7 +46,7 @@ func registerGuardian(v *extensions.Variant) { // enterprise management-cluster policy. Building the enterprise egress rules can // fail (proxy URL parsing); on failure we drop the policy entirely, matching the // core behavior of omitting it rather than installing a partial policy. -func modifyGuardianPolicy(ctx extensions.RenderContext, gpc render.GuardianPolicyExtensionContext, objs, del []client.Object) ([]client.Object, []client.Object) { +func modifyGuardianPolicy(rc extensions.RenderContext, gpc render.GuardianPolicyExtensionContext, objs, del []client.Object) ([]client.Object, []client.Object) { policy, ok := extensions.FindObject[*v3.NetworkPolicy](objs, render.GuardianPolicyName) if !ok { return objs, del @@ -204,7 +204,7 @@ func enterpriseGuardianPolicySpec(gpc render.GuardianPolicyExtensionContext) (v3 // objects: the secrets Role/RoleBinding and default UI settings, the // elasticsearch/kibana service ports, the management-cluster-request cluster // role rules (which replace the OSS rules), and the CA bundle env vars. -func modifyGuardian(ctx extensions.RenderContext, gc render.GuardianExtensionContext, objs, del []client.Object) ([]client.Object, []client.Object) { +func modifyGuardian(rc extensions.RenderContext, gc render.GuardianExtensionContext, objs, del []client.Object) ([]client.Object, []client.Object) { if role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, render.GuardianClusterRoleName); ok { role.Rules = guardianEnterpriseRules(gc) } diff --git a/pkg/enterprise/node.go b/pkg/enterprise/node.go index 60b02ccdff..43969b12a6 100644 --- a/pkg/enterprise/node.go +++ b/pkg/enterprise/node.go @@ -16,6 +16,7 @@ package enterprise import ( "fmt" + "slices" v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" appsv1 "k8s.io/api/apps/v1" @@ -47,23 +48,19 @@ const ( ) func registerNode(v *extensions.Variant) { - v.Image(render.ComponentNameNode, func(in *operatorv1.InstallationSpec) components.Component { - return components.ComponentTigeraNode - }) + v.Image(render.ComponentNameNode, components.ComponentTigeraNode) v.Modify(render.ComponentNameNode, modifyNode) // The node component renders the cni-plugins init container; its image // resolves through its own override key. - v.Image(render.ComponentNameCNIPlugins, func(in *operatorv1.InstallationSpec) components.Component { - return components.ComponentTigeraCNIPlugins - }) + v.Image(render.ComponentNameCNIPlugins, components.ComponentTigeraCNIPlugins) } // modifyNode layers Calico Enterprise behavior onto the rendered calico/node // objects: the extra RBAC rules, the node-metrics Service, and the Enterprise // daemonset configuration (flow/DNS log env, prometheus reporter, BGP metrics // readiness check, multi-interface mode, and the calico log volume). -func modifyNode(ctx extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { +func modifyNode(rc extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { if role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, render.CalicoNodeObjectName); ok { role.Rules = append(role.Rules, nodeEnterpriseRules()...) } @@ -78,10 +75,10 @@ func modifyNode(ctx extensions.RenderContext, objs, del []client.Object) ([]clie } if ds, ok := extensions.FindObject[*appsv1.DaemonSet](objs, common.NodeDaemonSetName); ok { - modifyNodeDaemonSet(ctx, ds) + modifyNodeDaemonSet(rc, ds) } - return append(objs, nodeMetricsService(ctx)), del + return append(objs, nodeMetricsService(rc)), del } // nodeEnterpriseRules are the additional cluster role rules calico/node needs in @@ -119,10 +116,10 @@ func nodeEnterpriseRules() []rbacv1.PolicyRule { // BGP metrics readiness check, and the prometheus reporter keypair mount. The // calico log volume is mounted by the base render for both variants, so it is // not handled here. -func modifyNodeDaemonSet(ctx extensions.RenderContext, ds *appsv1.DaemonSet) { +func modifyNodeDaemonSet(rc extensions.RenderContext, ds *appsv1.DaemonSet) { spec := &ds.Spec.Template.Spec - multiInterfaceMode := multiInterfaceModeEnv(ctx.Installation) + multiInterfaceMode := multiInterfaceModeEnv(rc.Installation) for i := range spec.InitContainers { if spec.InitContainers[i].Name == installCNIContainerName && multiInterfaceMode != nil { @@ -136,30 +133,30 @@ func modifyNodeDaemonSet(ctx extensions.RenderContext, ds *appsv1.DaemonSet) { continue } - c.Env = append(c.Env, nodeEnterpriseEnv(ctx)...) + c.Env = append(c.Env, nodeEnterpriseEnv(rc)...) // Add the BGP metrics readiness check, but only when the base render kept // the bird readiness check (i.e. BGP is in use and we're not on VPP). - if c.ReadinessProbe != nil && c.ReadinessProbe.Exec != nil && containsString(c.ReadinessProbe.Exec.Command, "--bird-ready") { + if c.ReadinessProbe != nil && c.ReadinessProbe.Exec != nil && slices.Contains(c.ReadinessProbe.Exec.Command, "--bird-ready") { c.ReadinessProbe.Exec.Command = append(c.ReadinessProbe.Exec.Command, "--bgp-metrics-ready") } } - mountNodePrometheusTLS(ctx, ds) + mountNodePrometheusTLS(rc, ds) } // mountNodePrometheusTLS mounts the node prometheus reporter keypair onto the // daemonset: the volume, the calico-node volume mount, the cert-management init // container (when in use), and the pod hash annotation that rolls the pods on // cert rotation. The keypair has cluster side effects, so the enterprise setup -// creates it and hands it in via ctx rather than the modifier building it. In +// creates it and hands it in via rc rather than the modifier building it. In // core (calico) the keypair is never created, so the base node render carries // no prometheus mount at all. -func mountNodePrometheusTLS(ctx extensions.RenderContext, ds *appsv1.DaemonSet) { - if ctx.NodePrometheusTLS == nil { +func mountNodePrometheusTLS(rc extensions.RenderContext, ds *appsv1.DaemonSet) { + if rc.NodePrometheusTLS == nil { return } - tls := ctx.NodePrometheusTLS + tls := rc.NodePrometheusTLS spec := &ds.Spec.Template.Spec spec.Volumes = append(spec.Volumes, tls.Volume()) @@ -183,10 +180,10 @@ func mountNodePrometheusTLS(ctx extensions.RenderContext, ds *appsv1.DaemonSet) // nodeEnterpriseEnv is the Enterprise felix configuration added to the // calico/node container. -func nodeEnterpriseEnv(ctx extensions.RenderContext) []corev1.EnvVar { +func nodeEnterpriseEnv(rc extensions.RenderContext) []corev1.EnvVar { env := []corev1.EnvVar{ {Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, - {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: fmt.Sprintf("%d", nodeReporterPort(ctx.FelixConfiguration))}, + {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: fmt.Sprintf("%d", nodeReporterPort(rc.FelixConfiguration))}, {Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, {Name: "FELIX_FLOWLOGSFILEINCLUDELABELS", Value: "true"}, {Name: "FELIX_FLOWLOGSFILEINCLUDEPOLICIES", Value: "true"}, @@ -197,15 +194,15 @@ func nodeEnterpriseEnv(ctx extensions.RenderContext) []corev1.EnvVar { {Name: "FELIX_DNSLOGSFILEPERNODELIMIT", Value: "1000"}, } - if mode := multiInterfaceModeEnv(ctx.Installation); mode != nil { + if mode := multiInterfaceModeEnv(rc.Installation); mode != nil { env = append(env, *mode) } - if ctx.NodePrometheusTLS != nil && ctx.TrustedBundle != nil { + if rc.NodePrometheusTLS != nil && rc.TrustedBundle != nil { env = append(env, - corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERCERTFILE", Value: ctx.NodePrometheusTLS.VolumeMountCertificateFilePath()}, - corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERKEYFILE", Value: ctx.NodePrometheusTLS.VolumeMountKeyFilePath()}, - corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERCAFILE", Value: ctx.TrustedBundle.MountPath()}, + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERCERTFILE", Value: rc.NodePrometheusTLS.VolumeMountCertificateFilePath()}, + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERKEYFILE", Value: rc.NodePrometheusTLS.VolumeMountKeyFilePath()}, + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERCAFILE", Value: rc.TrustedBundle.MountPath()}, ) } @@ -222,10 +219,10 @@ func multiInterfaceModeEnv(install *operatorv1.InstallationSpec) *corev1.EnvVar } // nodeMetricsService builds the enterprise-only calico-node-metrics Service. -func nodeMetricsService(ctx extensions.RenderContext) *corev1.Service { - reporterPort := nodeReporterPort(ctx.FelixConfiguration) - felixPort := felixMetricsPort(ctx.FelixConfiguration) - felixEnabled := ctx.FelixConfiguration != nil && utils.IsFelixPrometheusMetricsEnabled(ctx.FelixConfiguration) +func nodeMetricsService(rc extensions.RenderContext) *corev1.Service { + reporterPort := nodeReporterPort(rc.FelixConfiguration) + felixPort := felixMetricsPort(rc.FelixConfiguration) + felixEnabled := rc.FelixConfiguration != nil && utils.IsFelixPrometheusMetricsEnabled(rc.FelixConfiguration) ports := []corev1.ServicePort{ { @@ -283,12 +280,3 @@ func felixMetricsPort(fc *v3.FelixConfiguration) int { } return defaultFelixMetricsPort } - -func containsString(s []string, v string) bool { - for _, x := range s { - if x == v { - return true - } - } - return false -} diff --git a/pkg/enterprise/typha.go b/pkg/enterprise/typha.go index 77ce79000c..3786e7be07 100644 --- a/pkg/enterprise/typha.go +++ b/pkg/enterprise/typha.go @@ -28,7 +28,7 @@ func registerTypha(v *extensions.Variant) { v.Modify(render.ComponentNameTypha, modifyTypha) } -func modifyTypha(ctx extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { +func modifyTypha(rc extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { if role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, "calico-typha"); ok { role.Rules = append(role.Rules, rbacv1.PolicyRule{ APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, @@ -47,7 +47,7 @@ func modifyTypha(ctx extensions.RenderContext, objs, del []client.Object) ([]cli } if dep, ok := extensions.FindObject[*appsv1.Deployment](objs, "calico-typha"); ok { - net := ctx.Installation.CalicoNetwork + net := rc.Installation.CalicoNetwork if net != nil && net.MultiInterfaceMode != nil { for i := range dep.Spec.Template.Spec.Containers { if dep.Spec.Template.Spec.Containers[i].Name == render.TyphaContainerName { diff --git a/pkg/enterprise/windows.go b/pkg/enterprise/windows.go index 2b2d43753a..d00e7da88d 100644 --- a/pkg/enterprise/windows.go +++ b/pkg/enterprise/windows.go @@ -36,12 +36,8 @@ import ( var windowsNodeContainers = map[string]bool{"felix": true, "node": true, "confd": true} func registerWindows(v *extensions.Variant) { - v.Image(render.ComponentNameWindowsNodeImg, func(*operatorv1.InstallationSpec) components.Component { - return components.ComponentTigeraNodeWindows - }) - v.Image(render.ComponentNameWindowsCNIImg, func(*operatorv1.InstallationSpec) components.Component { - return components.ComponentTigeraCNIWindows - }) + v.Image(render.ComponentNameWindowsNodeImg, components.ComponentTigeraNodeWindows) + v.Image(render.ComponentNameWindowsCNIImg, components.ComponentTigeraCNIWindows) extensions.RegisterModifier(v, render.ComponentNameWindows, modifyWindows) } @@ -49,15 +45,15 @@ func registerWindows(v *extensions.Variant) { // calico-node-windows objects: the node-metrics Service and the Enterprise // daemonset configuration (flow/DNS log env, prometheus reporter, trusted DNS // servers, the calico log volume, and the prometheus reporter keypair mount). -func modifyWindows(ctx extensions.RenderContext, wc render.WindowsExtensionContext, objs, del []client.Object) ([]client.Object, []client.Object) { +func modifyWindows(rc extensions.RenderContext, wc render.WindowsExtensionContext, objs, del []client.Object) ([]client.Object, []client.Object) { if ds, ok := extensions.FindObject[*appsv1.DaemonSet](objs, common.WindowsDaemonSetName); ok { - modifyWindowsDaemonSet(ctx, wc, ds) + modifyWindowsDaemonSet(rc, wc, ds) } return append(objs, windowsNodeMetricsService(wc)), del } -func modifyWindowsDaemonSet(ctx extensions.RenderContext, wc render.WindowsExtensionContext, ds *appsv1.DaemonSet) { +func modifyWindowsDaemonSet(rc extensions.RenderContext, wc render.WindowsExtensionContext, ds *appsv1.DaemonSet) { dirOrCreate := corev1.HostPathDirectoryOrCreate spec := &ds.Spec.Template.Spec @@ -72,7 +68,7 @@ func modifyWindowsDaemonSet(ctx extensions.RenderContext, wc render.WindowsExten continue } - c.Env = append(c.Env, windowsEnterpriseEnv(ctx, wc)...) + c.Env = append(c.Env, windowsEnterpriseEnv(rc, wc)...) // Enterprise mounts the calico log directory in place of the OSS CNI log // directory, so drop the OSS mount before adding the enterprise one. @@ -85,7 +81,7 @@ func modifyWindowsDaemonSet(ctx extensions.RenderContext, wc render.WindowsExten // windowsEnterpriseEnv is the Enterprise felix configuration added to the // calico-node-windows containers. -func windowsEnterpriseEnv(ctx extensions.RenderContext, wc render.WindowsExtensionContext) []corev1.EnvVar { +func windowsEnterpriseEnv(rc extensions.RenderContext, wc render.WindowsExtensionContext) []corev1.EnvVar { env := []corev1.EnvVar{ {Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: fmt.Sprintf("%d", wc.NodeReporterMetricsPort)}, @@ -108,7 +104,7 @@ func windowsEnterpriseEnv(ctx extensions.RenderContext, wc render.WindowsExtensi } // Providers without a kube-dns service need a non-default trusted DNS server. - switch ctx.Installation.KubernetesProvider { + switch rc.Installation.KubernetesProvider { case operatorv1.ProviderOpenShift: env = append(env, corev1.EnvVar{Name: "FELIX_DNSTRUSTEDSERVERS", Value: "k8s-service:openshift-dns/dns-default"}) case operatorv1.ProviderRKE2: diff --git a/pkg/extensions/controllerextension.go b/pkg/extensions/controllerextension.go index ae22f5fd40..fada677689 100644 --- a/pkg/extensions/controllerextension.go +++ b/pkg/extensions/controllerextension.go @@ -22,20 +22,18 @@ import ( "github.com/tigera/operator/pkg/controller/certificatemanager" ) -// ControllerExtension is a variant's controller-side reconcile hook. The -// installation controller calls it to do the work core can't: reject -// unsupported configuration (Validate) and create the controller-side artifacts -// - certificates, trusted bundle additions - that feed the render context -// (ExtendContext). A variant registers at most one; the core operator registers -// none and runs with the base behavior. +// ControllerExtension extends a controller's reconcile: it validates the +// configuration and builds the RenderContext the render phase consumes. The core +// operator registers none and runs with the base behavior; an extension build +// registers one. type ControllerExtension interface { - // Validate rejects configuration the variant does not support, before any + // Validate rejects configuration the extension does not support, before any // rendering happens. Validate(cc ControllerContext) error - // ExtendContext does the controller-side work the render modifiers can't - // (creating certificates, extending the trusted bundle) and returns the - // RenderContext those modifiers read, or an error that aborts the reconcile. + // ExtendContext does the controller-side reconcile work the render phase + // cannot, returning the RenderContext the render phase consumes, or an error + // that aborts the reconcile. ExtendContext(cc ControllerContext) (RenderContext, error) } diff --git a/pkg/extensions/decorate_helpers_test.go b/pkg/extensions/decorate_helpers_test.go index 86afb71df0..82dc31b601 100644 --- a/pkg/extensions/decorate_helpers_test.go +++ b/pkg/extensions/decorate_helpers_test.go @@ -59,13 +59,13 @@ func (s stubExtComponent) ExtensionContext() any { // applyExtensions decorates a stub component holding the given objects with the // extension registered under key, then renders it. For a modifier that needs the // component's typed config, use applyExtensionsWithContext. -func applyExtensions(s *extensions.Set, key string, ctx extensions.RenderContext, create, del []client.Object) ([]client.Object, []client.Object) { - return applyExtensionsWithContext(s, key, ctx, nil, create, del) +func applyExtensions(s *extensions.Set, key string, rc extensions.RenderContext, create, del []client.Object) ([]client.Object, []client.Object) { + return applyExtensionsWithContext(s, key, rc, nil, create, del) } // applyExtensionsWithContext is applyExtensions for a modifier that reads the // component's typed config: extCtx is delivered as the stub's ExtensionContext. -func applyExtensionsWithContext(s *extensions.Set, key string, ctx extensions.RenderContext, extCtx any, create, del []client.Object) ([]client.Object, []client.Object) { +func applyExtensionsWithContext(s *extensions.Set, key string, rc extensions.RenderContext, extCtx any, create, del []client.Object) ([]client.Object, []client.Object) { stub := stubExtComponent{key: key, extCtx: extCtx, create: create, delete: del} - return s.Decorate(stub, ctx).Objects() + return s.Decorate(stub, rc).Objects() } diff --git a/pkg/extensions/extension.go b/pkg/extensions/extension.go index 3e00227624..106f1ee8bb 100644 --- a/pkg/extensions/extension.go +++ b/pkg/extensions/extension.go @@ -24,7 +24,7 @@ import ( // append objects to delete (e.g. to clean up resources a prior variant left // behind). It runs only for the variant it is registered under, so it need not // re-check the variant. -type Modifier func(ctx RenderContext, create, delete []client.Object) (newCreate, newDelete []client.Object) +type Modifier func(rc RenderContext, create, delete []client.Object) (newCreate, newDelete []client.Object) // FindObject returns the first object of type T with the given name. func FindObject[T client.Object](objs []client.Object, name string) (T, bool) { diff --git a/pkg/extensions/image.go b/pkg/extensions/image.go deleted file mode 100644 index a551d1e76f..0000000000 --- a/pkg/extensions/image.go +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright (c) 2026 Tigera, Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package extensions - -import ( - "github.com/tigera/operator/pkg/imageoverride" -) - -// ImageOverride returns the component image to use for an installation. It is -// the Image field of an Extension. An override runs only for the variant it was -// registered under, so it need not re-check the variant. -type ImageOverride = imageoverride.Override diff --git a/pkg/extensions/image_test.go b/pkg/extensions/image_test.go index 03d4abbce2..736f110ca3 100644 --- a/pkg/extensions/image_test.go +++ b/pkg/extensions/image_test.go @@ -27,9 +27,7 @@ var _ = Describe("image overrides", func() { var s *extensions.Set BeforeEach(func() { s = extensions.NewSet() - s.Variant(operatorv1.CalicoEnterprise).Image("node", func(in *operatorv1.InstallationSpec) components.Component { - return components.ComponentTigeraNode - }) + s.Variant(operatorv1.CalicoEnterprise).Image("node", components.ComponentTigeraNode) }) It("uses the override registered for the installation variant", func() { diff --git a/pkg/extensions/variant.go b/pkg/extensions/variant.go index 175463d4e8..36d4d8b96c 100644 --- a/pkg/extensions/variant.go +++ b/pkg/extensions/variant.go @@ -19,6 +19,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/imageoverride" "github.com/tigera/operator/pkg/render" ) @@ -37,7 +38,7 @@ type Variant struct { // decorator wraps a base component, returning one whose Objects() are augmented // by a registered modifier. -type decorator func(base render.Component, ctx RenderContext) render.Component +type decorator func(base render.Component, rc RenderContext) render.Component // Controller registers the variant's controller-side extension. A variant has // at most one; registering replaces any prior one. @@ -46,16 +47,16 @@ func (v *Variant) Controller(c ControllerExtension) { } // Image registers an image override for the named component. -func (v *Variant) Image(component string, fn ImageOverride) { - v.images.Register(v.variant, component, fn) +func (v *Variant) Image(component string, image components.Component) { + v.images.Register(v.variant, component, image) } // Modify registers a modifier for a component that needs no per-component // config. For components whose modifier needs the component's own typed config, // use RegisterModifier. func (v *Variant) Modify(component string, m Modifier) { - v.modifiers[component] = func(base render.Component, ctx RenderContext) render.Component { - return &decoratedComponent{Component: base, ctx: ctx, modify: m} + v.modifiers[component] = func(base render.Component, rc RenderContext) render.Component { + return &decoratedComponent{Component: base, rc: rc, modify: m} } } @@ -64,10 +65,12 @@ func (v *Variant) Modify(component string, m Modifier) { // render.ExtensionContextProvider; RegisterModifier asserts it to Cfg once, here, // and hands the typed value to modify - so the modifier body needs no assertion. // It is a free function because Go has no generic methods. -func RegisterModifier[Cfg any](v *Variant, component string, - modify func(ctx RenderContext, cfg Cfg, create, delete []client.Object) ([]client.Object, []client.Object), +func RegisterModifier[Cfg any]( + v *Variant, + component string, + modify func(rc RenderContext, cfg Cfg, create, delete []client.Object) ([]client.Object, []client.Object), ) { - v.modifiers[component] = func(base render.Component, ctx RenderContext) render.Component { + v.modifiers[component] = func(base render.Component, rc RenderContext) render.Component { provider, ok := base.(render.ExtensionContextProvider) if !ok { logrus.Errorf("BUG: component %q has a registered modifier but provides no extension context; leaving it unmodified", component) @@ -79,17 +82,17 @@ func RegisterModifier[Cfg any](v *Variant, component string, logrus.Errorf("BUG: component %q extension context is %T, want %T; leaving it unmodified", component, provider.ExtensionContext(), want) return base } - bound := func(ctx RenderContext, create, delete []client.Object) ([]client.Object, []client.Object) { - return modify(ctx, cfg, create, delete) + bound := func(rc RenderContext, create, delete []client.Object) ([]client.Object, []client.Object) { + return modify(rc, cfg, create, delete) } - return &decoratedComponent{Component: base, ctx: ctx, modify: bound} + return &decoratedComponent{Component: base, rc: rc, modify: bound} } } // decorate wraps component with the modifier registered for its extension key, // or returns it unchanged when the component exposes no extension point or none // is registered. Nil-safe. -func (v *Variant) decorate(component render.Component, ctx RenderContext) render.Component { +func (v *Variant) decorate(component render.Component, rc RenderContext) render.Component { if v == nil { return component } @@ -101,7 +104,7 @@ func (v *Variant) decorate(component render.Component, ctx RenderContext) render if !ok { return component } - return build(component, ctx) + return build(component, rc) } // validate runs the controller extension's validation, or nil when the variant @@ -128,11 +131,11 @@ func (v *Variant) extendContext(cc ControllerContext) (RenderContext, error) { // and Ready delegate to the base; only Objects is augmented. type decoratedComponent struct { render.Component - ctx RenderContext + rc RenderContext modify Modifier } func (d *decoratedComponent) Objects() ([]client.Object, []client.Object) { create, del := d.Component.Objects() - return d.modify(d.ctx, create, del) + return d.modify(d.rc, create, del) } diff --git a/pkg/imageoverride/imageoverride.go b/pkg/imageoverride/imageoverride.go index 5c981c0c07..ee4c48d559 100644 --- a/pkg/imageoverride/imageoverride.go +++ b/pkg/imageoverride/imageoverride.go @@ -22,9 +22,6 @@ import ( "github.com/tigera/operator/pkg/components" ) -// Override selects the component image to use for an installation. -type Override func(in *operatorv1.InstallationSpec) components.Component - type overrideKey struct { variant operatorv1.ProductVariant key string @@ -33,19 +30,21 @@ type overrideKey struct { // Overrides maps a component (keyed by variant) to the image it should resolve // to, letting a variant swap a component's image without the render package // branching on variant. The render component holds one and resolves through it. +// Registry, image path, and FIPS handling are applied downstream in the render +// package, so an override only picks which component. type Overrides struct { - m map[overrideKey]Override + m map[overrideKey]components.Component } // New returns an empty Overrides. func New() *Overrides { - return &Overrides{m: map[overrideKey]Override{}} + return &Overrides{m: map[overrideKey]components.Component{}} } -// Register stores fn under key for the given variant. The key is the render +// Register stores image under key for the given variant. The key is the render // component's image identifier (e.g. "node"). -func (o *Overrides) Register(variant operatorv1.ProductVariant, key string, fn Override) { - o.m[overrideKey{variant, key}] = fn +func (o *Overrides) Register(variant operatorv1.ProductVariant, key string, image components.Component) { + o.m[overrideKey{variant, key}] = image } // Resolve returns the override registered for key under the installation's @@ -55,8 +54,8 @@ func (o *Overrides) Resolve(key string, def components.Component, in *operatorv1 if o == nil || in == nil { return def } - if fn, ok := o.m[overrideKey{in.Variant, key}]; ok { - return fn(in) + if image, ok := o.m[overrideKey{in.Variant, key}]; ok { + return image } return def } diff --git a/pkg/render/decorate_helpers_test.go b/pkg/render/decorate_helpers_test.go index c748f56295..75f0587276 100644 --- a/pkg/render/decorate_helpers_test.go +++ b/pkg/render/decorate_helpers_test.go @@ -59,13 +59,13 @@ func (s stubExtComponent) ExtensionContext() any { // applyExtensions decorates a stub component holding the given objects with the // extension registered under key, then renders it. For a modifier that needs the // component's typed config, use applyExtensionsWithContext. -func applyExtensions(s *extensions.Set, key string, ctx extensions.RenderContext, create, del []client.Object) ([]client.Object, []client.Object) { - return applyExtensionsWithContext(s, key, ctx, nil, create, del) +func applyExtensions(s *extensions.Set, key string, rc extensions.RenderContext, create, del []client.Object) ([]client.Object, []client.Object) { + return applyExtensionsWithContext(s, key, rc, nil, create, del) } // applyExtensionsWithContext is applyExtensions for a modifier that reads the // component's typed config: extCtx is delivered as the stub's ExtensionContext. -func applyExtensionsWithContext(s *extensions.Set, key string, ctx extensions.RenderContext, extCtx any, create, del []client.Object) ([]client.Object, []client.Object) { +func applyExtensionsWithContext(s *extensions.Set, key string, rc extensions.RenderContext, extCtx any, create, del []client.Object) ([]client.Object, []client.Object) { stub := stubExtComponent{key: key, extCtx: extCtx, create: create, delete: del} - return s.Decorate(stub, ctx).Objects() + return s.Decorate(stub, rc).Objects() } From 76eefc3cc7b0d88af8a4d2801bb0661e5e39d70d Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Thu, 18 Jun 2026 11:48:27 -0700 Subject: [PATCH 31/38] Carry extension-produced data in an opaque RenderContext slot RenderContext no longer names the enterprise node prometheus keypair. It has an opaque Extension slot the controller extension fills and its own modifiers assert back out. The installation extension stashes the keypair there for the node modifier and returns it as one the controller should manage, so the controller no longer references the enterprise keypair in its cert-management and warning wiring. --- .../installation/core_controller.go | 16 +++++--- pkg/enterprise/installation.go | 36 +++++++++++++---- pkg/enterprise/installation_test.go | 10 +++-- pkg/enterprise/node.go | 11 ++--- pkg/extensions/controllerextension.go | 9 +++-- pkg/extensions/controllerextension_test.go | 19 ++++----- pkg/extensions/rendercontext.go | 20 +++++----- pkg/extensions/set.go | 8 ++-- pkg/extensions/variant.go | 7 ++-- pkg/render/node_enterprise_test.go | 40 +++++++++++++------ 10 files changed, 113 insertions(+), 63 deletions(-) diff --git a/pkg/controller/installation/core_controller.go b/pkg/controller/installation/core_controller.go index 47fdea89c1..4dd3d8554c 100644 --- a/pkg/controller/installation/core_controller.go +++ b/pkg/controller/installation/core_controller.go @@ -1224,7 +1224,7 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile r.status.SetDegraded(operatorv1.ResourceValidationError, "Invalid installation configuration", err, reqLogger) return reconcile.Result{}, err } - renderCtx, err := r.opts.Extensions.ExtendContext(cc) + renderCtx, managedKeyPairs, err := r.opts.Extensions.ExtendContext(cc) if err != nil { r.status.SetDegraded(operatorv1.ResourceCreateError, "Error preparing installation extension", err, reqLogger) return reconcile.Result{}, err @@ -1367,7 +1367,6 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile keyPairOptions := []rcertificatemanagement.KeyPairOption{ rcertificatemanagement.NewKeyPairOption(typhaNodeTLS.NodeSecret, true, true), - rcertificatemanagement.NewKeyPairOption(renderCtx.NodePrometheusTLS, true, true), rcertificatemanagement.NewKeyPairOption(typhaNodeTLS.TyphaSecret, true, true), rcertificatemanagement.NewKeyPairOption(typhaNodeTLS.TyphaSecretNonClusterHost, true, true), rcertificatemanagement.NewKeyPairOption(kubeControllerTLS, true, true), @@ -1375,6 +1374,10 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile // render skips nil key pairs. rcertificatemanagement.NewKeyPairOption(wafWebhookTLS, true, true), } + // Manage any key pairs the variant extension created controller-side. + for _, kp := range managedKeyPairs { + keyPairOptions = append(keyPairOptions, rcertificatemanagement.NewKeyPairOption(kp, true, true)) + } components = append(components, rcertificatemanagement.CertificateManagement(&rcertificatemanagement.Config{ @@ -1795,13 +1798,16 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile r.status.ReadyToMonitor() // Check BYO certificate expiry warnings and propagate them to the status manager. - certificatemanagement.CheckKeyPairWarnings(map[string]certificatemanagement.KeyPairInterface{ + keyPairWarnings := map[string]certificatemanagement.KeyPairInterface{ render.TyphaTLSSecretName: typhaNodeTLS.TyphaSecret, render.NodeTLSSecretName: typhaNodeTLS.NodeSecret, render.TyphaTLSSecretName + render.TyphaNonClusterHostSuffix: typhaNodeTLS.TyphaSecretNonClusterHost, - render.NodePrometheusTLSServerSecret: renderCtx.NodePrometheusTLS, kubecontrollers.KubeControllerPrometheusTLSSecret: kubeControllerTLS, - }, r.status) + } + for _, kp := range managedKeyPairs { + keyPairWarnings[kp.GetName()] = kp + } + certificatemanagement.CheckKeyPairWarnings(keyPairWarnings, r.status) // We can clear the degraded state now since as far as we know everything is in order. r.status.ClearDegraded() diff --git a/pkg/enterprise/installation.go b/pkg/enterprise/installation.go index 57ad67476c..4eb89c2282 100644 --- a/pkg/enterprise/installation.go +++ b/pkg/enterprise/installation.go @@ -24,12 +24,27 @@ import ( "github.com/tigera/operator/pkg/render" relasticsearch "github.com/tigera/operator/pkg/render/common/elasticsearch" "github.com/tigera/operator/pkg/render/monitor" + "github.com/tigera/operator/pkg/tls/certificatemanagement" ) // controllerExtension is the Calico Enterprise controller-side hook for the // installation controller. type controllerExtension struct{} +// installationRenderData is the controller-produced data the installation +// extension hands to its modifiers through RenderContext.Extension. The node +// modifier type-asserts it back out. +type installationRenderData struct { + nodePrometheusTLS certificatemanagement.KeyPairInterface +} + +// installationData pulls the installation extension's render data back out of the +// render context, returning the zero value when none is set. +func installationData(rc extensions.RenderContext) installationRenderData { + data, _ := rc.Extension.(installationRenderData) + return data +} + // Validate rejects installation config Calico Enterprise does not support. func (controllerExtension) Validate(cc extensions.ControllerContext) error { // Reject the unsupported zero reporter port. The port value itself is derived @@ -41,9 +56,10 @@ func (controllerExtension) Validate(cc extensions.ControllerContext) error { } // ExtendContext does the controller-side work the modifiers can't: creating and -// fetching the certificates that feed the trusted bundle, returning the render -// context with the produced node prometheus keypair layered on. -func (controllerExtension) ExtendContext(cc extensions.ControllerContext) (extensions.RenderContext, error) { +// fetching the certificates that feed the trusted bundle. It returns the render +// context carrying the produced node prometheus keypair, and that keypair as one +// the controller should manage. +func (controllerExtension) ExtendContext(cc extensions.ControllerContext) (extensions.RenderContext, []certificatemanagement.KeyPairInterface, error) { rc := cc.RenderContext nodePrometheusTLS, err := cc.CertificateManager.GetOrCreateKeyPair( @@ -53,16 +69,16 @@ func (controllerExtension) ExtendContext(cc extensions.ControllerContext) (exten dns.GetServiceDNSNames(render.CalicoNodeMetricsService, common.CalicoNamespace, cc.ClusterDomain), ) if err != nil { - return rc, fmt.Errorf("error creating node prometheus TLS certificate: %w", err) + return rc, nil, fmt.Errorf("error creating node prometheus TLS certificate: %w", err) } if nodePrometheusTLS != nil { cc.TrustedBundle.AddCertificates(nodePrometheusTLS) } - rc.NodePrometheusTLS = nodePrometheusTLS + rc.Extension = installationRenderData{nodePrometheusTLS: nodePrometheusTLS} prometheusClientCert, err := cc.CertificateManager.GetCertificate(cc.Client, monitor.PrometheusClientTLSSecretName, common.OperatorNamespace()) if err != nil { - return rc, fmt.Errorf("unable to fetch prometheus certificate: %w", err) + return rc, nil, fmt.Errorf("unable to fetch prometheus certificate: %w", err) } if prometheusClientCert != nil { cc.TrustedBundle.AddCertificates(prometheusClientCert) @@ -70,11 +86,15 @@ func (controllerExtension) ExtendContext(cc extensions.ControllerContext) (exten esgwCertificate, err := cc.CertificateManager.GetCertificate(cc.Client, relasticsearch.PublicCertSecret, common.OperatorNamespace()) if err != nil { - return rc, fmt.Errorf("failed to retrieve / validate %s: %w", relasticsearch.PublicCertSecret, err) + return rc, nil, fmt.Errorf("failed to retrieve / validate %s: %w", relasticsearch.PublicCertSecret, err) } if esgwCertificate != nil { cc.TrustedBundle.AddCertificates(esgwCertificate) } - return rc, nil + var managed []certificatemanagement.KeyPairInterface + if nodePrometheusTLS != nil { + managed = append(managed, nodePrometheusTLS) + } + return rc, managed, nil } diff --git a/pkg/enterprise/installation_test.go b/pkg/enterprise/installation_test.go index f968306b96..bd790bc161 100644 --- a/pkg/enterprise/installation_test.go +++ b/pkg/enterprise/installation_test.go @@ -29,6 +29,7 @@ import ( "github.com/tigera/operator/pkg/controller/certificatemanager" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/render" ) var _ = Describe("installation controller extension", func() { @@ -43,15 +44,16 @@ var _ = Describe("installation controller extension", func() { }) It("creates the node prometheus keypair for the enterprise variant", func() { - rc, err := ext.ExtendContext(newControllerContext(operatorv1.CalicoEnterprise)) + _, managed, err := ext.ExtendContext(newControllerContext(operatorv1.CalicoEnterprise)) Expect(err).NotTo(HaveOccurred()) - Expect(rc.NodePrometheusTLS).NotTo(BeNil()) + Expect(managed).To(HaveLen(1), "expected the node prometheus keypair to be managed") + Expect(managed[0].GetName()).To(Equal(render.NodePrometheusTLSServerSecret)) }) It("is a no-op for the Calico variant", func() { - rc, err := ext.ExtendContext(newControllerContext(operatorv1.Calico)) + _, managed, err := ext.ExtendContext(newControllerContext(operatorv1.Calico)) Expect(err).NotTo(HaveOccurred()) - Expect(rc.NodePrometheusTLS).To(BeNil()) + Expect(managed).To(BeEmpty()) }) }) diff --git a/pkg/enterprise/node.go b/pkg/enterprise/node.go index 43969b12a6..08f2ecf7f8 100644 --- a/pkg/enterprise/node.go +++ b/pkg/enterprise/node.go @@ -153,10 +153,10 @@ func modifyNodeDaemonSet(rc extensions.RenderContext, ds *appsv1.DaemonSet) { // core (calico) the keypair is never created, so the base node render carries // no prometheus mount at all. func mountNodePrometheusTLS(rc extensions.RenderContext, ds *appsv1.DaemonSet) { - if rc.NodePrometheusTLS == nil { + tls := installationData(rc).nodePrometheusTLS + if tls == nil { return } - tls := rc.NodePrometheusTLS spec := &ds.Spec.Template.Spec spec.Volumes = append(spec.Volumes, tls.Volume()) @@ -181,6 +181,7 @@ func mountNodePrometheusTLS(rc extensions.RenderContext, ds *appsv1.DaemonSet) { // nodeEnterpriseEnv is the Enterprise felix configuration added to the // calico/node container. func nodeEnterpriseEnv(rc extensions.RenderContext) []corev1.EnvVar { + tls := installationData(rc).nodePrometheusTLS env := []corev1.EnvVar{ {Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: fmt.Sprintf("%d", nodeReporterPort(rc.FelixConfiguration))}, @@ -198,10 +199,10 @@ func nodeEnterpriseEnv(rc extensions.RenderContext) []corev1.EnvVar { env = append(env, *mode) } - if rc.NodePrometheusTLS != nil && rc.TrustedBundle != nil { + if tls != nil && rc.TrustedBundle != nil { env = append(env, - corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERCERTFILE", Value: rc.NodePrometheusTLS.VolumeMountCertificateFilePath()}, - corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERKEYFILE", Value: rc.NodePrometheusTLS.VolumeMountKeyFilePath()}, + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERCERTFILE", Value: tls.VolumeMountCertificateFilePath()}, + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERKEYFILE", Value: tls.VolumeMountKeyFilePath()}, corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERCAFILE", Value: rc.TrustedBundle.MountPath()}, ) } diff --git a/pkg/extensions/controllerextension.go b/pkg/extensions/controllerextension.go index fada677689..05a176bdd9 100644 --- a/pkg/extensions/controllerextension.go +++ b/pkg/extensions/controllerextension.go @@ -20,6 +20,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/tls/certificatemanagement" ) // ControllerExtension extends a controller's reconcile: it validates the @@ -32,9 +33,11 @@ type ControllerExtension interface { Validate(cc ControllerContext) error // ExtendContext does the controller-side reconcile work the render phase - // cannot, returning the RenderContext the render phase consumes, or an error - // that aborts the reconcile. - ExtendContext(cc ControllerContext) (RenderContext, error) + // cannot, returning the RenderContext the render phase consumes plus any + // keypairs the extension created that the controller should manage (add to + // certificate management and BYO-expiry warnings), or an error that aborts the + // reconcile. + ExtendContext(cc ControllerContext) (RenderContext, []certificatemanagement.KeyPairInterface, error) } // ControllerContext is the controller-phase context, the corollary to the diff --git a/pkg/extensions/controllerextension_test.go b/pkg/extensions/controllerextension_test.go index a0fc51275b..c38d4e5e02 100644 --- a/pkg/extensions/controllerextension_test.go +++ b/pkg/extensions/controllerextension_test.go @@ -22,6 +22,7 @@ import ( operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/tls/certificatemanagement" ) var _ = Describe("controller extension", func() { @@ -32,25 +33,25 @@ var _ = Describe("controller extension", func() { It("returns the base render context when the variant has no extension", func() { install := &operatorv1.InstallationSpec{Variant: operatorv1.Calico} - rc, err := s.ExtendContext(extensions.ControllerContext{ + rc, _, err := s.ExtendContext(extensions.ControllerContext{ RenderContext: extensions.RenderContext{Installation: install, ClusterDomain: "cluster.local"}, }) Expect(err).NotTo(HaveOccurred()) Expect(rc.Installation).To(BeIdenticalTo(install)) Expect(rc.ClusterDomain).To(Equal("cluster.local")) - Expect(rc.NodePrometheusTLS).To(BeNil()) + Expect(rc.Extension).To(BeNil()) }) It("runs the extension registered for the installation variant", func() { s.Variant(operatorv1.CalicoEnterprise).Controller(fakeController{}) - rc, err := s.ExtendContext(enterpriseContext()) + rc, _, err := s.ExtendContext(enterpriseContext()) Expect(err).NotTo(HaveOccurred()) Expect(rc.ClusterDomain).To(Equal("from-fake")) }) It("ignores an extension registered for a different variant", func() { s.Variant(operatorv1.CalicoEnterprise).Controller(fakeController{}) - rc, err := s.ExtendContext(extensions.ControllerContext{ + rc, _, err := s.ExtendContext(extensions.ControllerContext{ RenderContext: extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}, ClusterDomain: "real"}, }) Expect(err).NotTo(HaveOccurred()) @@ -59,7 +60,7 @@ var _ = Describe("controller extension", func() { It("surfaces the extension error", func() { s.Variant(operatorv1.CalicoEnterprise).Controller(fakeController{err: errors.New("boom")}) - _, err := s.ExtendContext(enterpriseContext()) + _, _, err := s.ExtendContext(enterpriseContext()) Expect(err).To(MatchError("boom")) }) @@ -72,7 +73,7 @@ var _ = Describe("controller extension", func() { var nilSet *extensions.Set cc := enterpriseContext() cc.ClusterDomain = "real" - rc, err := nilSet.ExtendContext(cc) + rc, _, err := nilSet.ExtendContext(cc) Expect(err).NotTo(HaveOccurred()) Expect(rc.ClusterDomain).To(Equal("real")) Expect(nilSet.Validate(cc)).NotTo(HaveOccurred()) @@ -96,9 +97,9 @@ func (f fakeController) Validate(_ extensions.ControllerContext) error { return f.validateErr } -func (f fakeController) ExtendContext(_ extensions.ControllerContext) (extensions.RenderContext, error) { +func (f fakeController) ExtendContext(_ extensions.ControllerContext) (extensions.RenderContext, []certificatemanagement.KeyPairInterface, error) { if f.err != nil { - return extensions.RenderContext{}, f.err + return extensions.RenderContext{}, nil, f.err } - return extensions.RenderContext{ClusterDomain: "from-fake"}, nil + return extensions.RenderContext{ClusterDomain: "from-fake"}, nil, nil } diff --git a/pkg/extensions/rendercontext.go b/pkg/extensions/rendercontext.go index 95daec43aa..3e092bf18c 100644 --- a/pkg/extensions/rendercontext.go +++ b/pkg/extensions/rendercontext.go @@ -22,11 +22,10 @@ import ( // RenderContext carries reconcile-derived inputs from controllers into render // modifiers. Core operator code never reads these fields - only registered -// modifiers do. Two kinds of value live here: -// - raw cluster state gathered generically (Installation, FelixConfiguration, -// ClusterDomain) that modifiers derive their own values from, and -// - controller-produced artifacts (TrustedBundle, NodePrometheusTLS) that can -// only be created controller-side because they have cluster side effects. +// modifiers do. It carries raw cluster state gathered generically (Installation, +// FelixConfiguration, ClusterDomain) that modifiers derive their own values from, +// the shared TrustedBundle, and an opaque Extension slot for controller-produced +// data specific to one extension. // // Per-component config a modifier needs but can't derive from these fields is // not carried here; it flows to the modifier as a typed argument (see @@ -39,9 +38,10 @@ type RenderContext struct { // TrustedBundle is the shared CA bundle for the calico-system namespace. TrustedBundle certificatemanagement.TrustedBundle - // NodePrometheusTLS is created by the enterprise controller extension (it has - // cluster side effects, so it can't be built in a modifier). The node modifier - // is its only consumer: it mounts the keypair onto the daemonset and sets the - // FELIX_PROMETHEUSREPORTER* certificate env vars. - NodePrometheusTLS certificatemanagement.KeyPairInterface + // Extension is opaque, extension-owned data that the controller extension + // produced for its own modifiers - typically an artifact that can only be + // created controller-side because it has cluster side effects (e.g. a keypair). + // The extension that set it type-asserts it back out in its modifiers; core + // code never reads it. Nil when no extension is active. + Extension any } diff --git a/pkg/extensions/set.go b/pkg/extensions/set.go index 42a76267ad..ac97f821d5 100644 --- a/pkg/extensions/set.go +++ b/pkg/extensions/set.go @@ -19,6 +19,7 @@ import ( "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/imageoverride" "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/tls/certificatemanagement" ) // Set is all the variant extensions the operator runs with, indexed by product @@ -88,11 +89,12 @@ func (s *Set) Validate(cc ControllerContext) error { } // ExtendContext runs the controller extension for the installation's variant and -// returns the resulting RenderContext, or the base render context when no +// returns the resulting RenderContext plus any keypairs the extension wants the +// controller to manage, or the base render context and no keypairs when no // extension is registered. Nil-safe. -func (s *Set) ExtendContext(cc ControllerContext) (RenderContext, error) { +func (s *Set) ExtendContext(cc ControllerContext) (RenderContext, []certificatemanagement.KeyPairInterface, error) { if cc.Installation == nil { - return cc.RenderContext, nil + return cc.RenderContext, nil, nil } return s.variant(cc.Installation.Variant).extendContext(cc) } diff --git a/pkg/extensions/variant.go b/pkg/extensions/variant.go index 36d4d8b96c..08c920bbc5 100644 --- a/pkg/extensions/variant.go +++ b/pkg/extensions/variant.go @@ -22,6 +22,7 @@ import ( "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/imageoverride" "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/tls/certificatemanagement" ) // Variant bundles everything that extends the core operator for one product @@ -117,10 +118,10 @@ func (v *Variant) validate(cc ControllerContext) error { } // extendContext runs the controller extension, or returns the base render -// context when the variant has none. Nil-safe. -func (v *Variant) extendContext(cc ControllerContext) (RenderContext, error) { +// context and no managed keypairs when the variant has none. Nil-safe. +func (v *Variant) extendContext(cc ControllerContext) (RenderContext, []certificatemanagement.KeyPairInterface, error) { if v == nil || v.controller == nil { - return cc.RenderContext, nil + return cc.RenderContext, nil, nil } return v.controller.ExtendContext(cc) } diff --git a/pkg/render/node_enterprise_test.go b/pkg/render/node_enterprise_test.go index c9c32b69fd..9ea5db8d2e 100644 --- a/pkg/render/node_enterprise_test.go +++ b/pkg/render/node_enterprise_test.go @@ -15,6 +15,8 @@ package render_test import ( + "context" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -34,6 +36,7 @@ import ( "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" rmeta "github.com/tigera/operator/pkg/render/common/meta" + "github.com/tigera/operator/pkg/tls/certificatemanagement" ) // These tests run the real node/typha render output through the registered @@ -43,11 +46,12 @@ import ( // matching because render renamed an object or container. var _ = Describe("node enterprise modifier integration", func() { var ( - cli client.Client - certManager certificatemanager.CertificateManager - typhaNodeTLS *render.TyphaNodeTLS - instance *operatorv1.InstallationSpec - renderCtx extensions.RenderContext + cli client.Client + certManager certificatemanager.CertificateManager + typhaNodeTLS *render.TyphaNodeTLS + instance *operatorv1.InstallationSpec + renderCtx extensions.RenderContext + nodePrometheusTLS certificatemanagement.KeyPairInterface ) nodeContainer := func(ds *appsv1.DaemonSet) *corev1.Container { @@ -69,7 +73,7 @@ var _ = Describe("node enterprise modifier integration", func() { Expect(err).NotTo(HaveOccurred()) typhaNodeTLS = getTyphaNodeTLS(cli, certManager) - nodePrometheusTLS, err := certManager.GetOrCreateKeyPair(cli, render.NodePrometheusTLSServerSecret, common.OperatorNamespace(), []string{"calico-node-metrics"}) + nodePrometheusTLS, err = certManager.GetOrCreateKeyPair(cli, render.NodePrometheusTLSServerSecret, common.OperatorNamespace(), []string{"calico-node-metrics"}) Expect(err).NotTo(HaveOccurred()) typhaNodeTLS.TrustedBundle.AddCertificates(nodePrometheusTLS) @@ -89,11 +93,21 @@ var _ = Describe("node enterprise modifier integration", func() { }, } - renderCtx = extensions.RenderContext{ - Installation: instance, - TrustedBundle: typhaNodeTLS.TrustedBundle, - NodePrometheusTLS: nodePrometheusTLS, + // Build the render context the way the controller does: run the enterprise + // controller extension, which stashes the node prometheus keypair in the + // context for the node modifier to read. + cc := extensions.ControllerContext{ + RenderContext: extensions.RenderContext{ + Installation: instance, + TrustedBundle: typhaNodeTLS.TrustedBundle, + ClusterDomain: dns.DefaultClusterDomain, + }, + Ctx: context.Background(), + Client: cli, + CertificateManager: certManager, } + renderCtx, _, err = ext.ExtendContext(cc) + Expect(err).NotTo(HaveOccurred()) }) // renderNodeObjects renders the real node component and applies the registered @@ -148,9 +162,9 @@ var _ = Describe("node enterprise modifier integration", func() { // The reporter cert env is wired from the NodePrometheusTLS keypair the // builder creates, and the modifier mounts that keypair onto the daemonset. Expect(c.Env).To(ContainElement(HaveField("Name", "FELIX_PROMETHEUSREPORTERCERTFILE"))) - Expect(ds.Spec.Template.Spec.Volumes).To(ContainElement(renderCtx.NodePrometheusTLS.Volume())) - Expect(c.VolumeMounts).To(ContainElement(renderCtx.NodePrometheusTLS.VolumeMount(rmeta.OSTypeLinux))) - Expect(ds.Spec.Template.Annotations).To(HaveKey(renderCtx.NodePrometheusTLS.HashAnnotationKey())) + Expect(ds.Spec.Template.Spec.Volumes).To(ContainElement(nodePrometheusTLS.Volume())) + Expect(c.VolumeMounts).To(ContainElement(nodePrometheusTLS.VolumeMount(rmeta.OSTypeLinux))) + Expect(ds.Spec.Template.Annotations).To(HaveKey(nodePrometheusTLS.HashAnnotationKey())) // BGP is enabled, so the bird readiness check is present and the modifier // adds the BGP metrics check. From 1d831fd9ac000768b0b553c8db8bf9dfb81501e3 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Thu, 18 Jun 2026 12:42:00 -0700 Subject: [PATCH 32/38] Decouple the windows controller from enterprise via a per-controller hook Controller extensions are now registered per controller (a ControllerName plus constants) and selected by ControllerContext.Controller, so each controller runs its own hook. The windows controller runs its hook to fetch the node prometheus keypair into the render context's Extension slot; its IsEnterprise branch is gone and WindowsConfiguration no longer carries the enterprise reporter port or keypair (the modifier derives the port from FelixConfiguration and reads the keypair from the slot). Rename the installation hook to coreControllerExtension. --- .../installation/core_controller.go | 7 +- .../installation/windows_controller.go | 72 ++++++++--------- pkg/enterprise/installation.go | 16 ++-- pkg/enterprise/installation_test.go | 1 + pkg/enterprise/node.go | 10 +++ pkg/enterprise/register.go | 3 +- pkg/enterprise/windows.go | 81 ++++++++++++++----- pkg/enterprise/windows_test.go | 39 ++++++--- pkg/extensions/controllerextension.go | 22 ++++- pkg/extensions/controllerextension_test.go | 11 ++- pkg/extensions/set.go | 15 ++-- pkg/extensions/variant.go | 32 ++++---- pkg/render/node_enterprise_test.go | 1 + pkg/render/render_test.go | 32 ++++---- pkg/render/windows.go | 34 ++------ pkg/render/windows_test.go | 10 +-- 16 files changed, 211 insertions(+), 175 deletions(-) diff --git a/pkg/controller/installation/core_controller.go b/pkg/controller/installation/core_controller.go index 4dd3d8554c..29fafcccf6 100644 --- a/pkg/controller/installation/core_controller.go +++ b/pkg/controller/installation/core_controller.go @@ -91,12 +91,6 @@ import ( const ( techPreviewFeatureSeccompApparmor = "tech-preview.operator.tigera.io/node-apparmor-profile" - - // defaultNodeReporterPort is the default port calico/node uses to report Calico - // Enterprise internal metrics. The Linux node path derives this in the - // enterprise node modifier; this copy serves the Windows controller, which - // still carries its enterprise logic inline. - defaultNodeReporterPort = 9081 ) const InstallationName string = "calico" @@ -1216,6 +1210,7 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile ClusterDomain: r.opts.ClusterDomain, TrustedBundle: typhaNodeTLS.TrustedBundle, }, + Controller: extensions.InstallationController, Ctx: ctx, Client: r.client, CertificateManager: certificateManager, diff --git a/pkg/controller/installation/windows_controller.go b/pkg/controller/installation/windows_controller.go index ad758dbbce..0e8152d85b 100644 --- a/pkg/controller/installation/windows_controller.go +++ b/pkg/controller/installation/windows_controller.go @@ -16,7 +16,6 @@ package installation import ( "context" - "errors" "fmt" "reflect" @@ -50,11 +49,9 @@ import ( "github.com/tigera/operator/pkg/controller/utils" "github.com/tigera/operator/pkg/controller/utils/imageset" "github.com/tigera/operator/pkg/ctrlruntime" - "github.com/tigera/operator/pkg/dns" "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" "github.com/tigera/operator/pkg/render/monitor" - "github.com/tigera/operator/pkg/tls/certificatemanagement" ) var logw = logf.Log.WithName("controller_windows") @@ -325,33 +322,6 @@ func (r *ReconcileWindows) Reconcile(ctx context.Context, request reconcile.Requ } } - // nodeReporterMetricsPort is a port used in Enterprise to host internal metrics. - // Operator is responsible for creating a service which maps to that port. - // Here, we'll check the default felixconfiguration to see if the user is specifying - // a non-default port, and use that value if they are. - nodeReporterMetricsPort := defaultNodeReporterPort - var nodePrometheusTLS certificatemanagement.KeyPairInterface - if instance.Spec.Variant.IsEnterprise() { - - // Determine the port to use for nodeReporter metrics. - if felixConfiguration.Spec.PrometheusReporterPort != nil { - nodeReporterMetricsPort = *felixConfiguration.Spec.PrometheusReporterPort - } - - if nodeReporterMetricsPort == 0 { - err := errors.New("felixConfiguration prometheusReporterPort=0 not supported") - r.status.SetDegraded(operatorv1.InvalidConfigurationError, "invalid metrics port", err, reqLogger) - return reconcile.Result{}, err - } - - // The key pair is created by the core controller, so if it isn't set, requeue to wait until it is - nodePrometheusTLS, err = certificateManager.GetKeyPair(r.client, render.NodePrometheusTLSServerSecret, common.OperatorNamespace(), dns.GetServiceDNSNames(render.WindowsNodeMetricsService, common.CalicoNamespace, r.opts.ClusterDomain)) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceCreateError, "Error getting TLS certificate", err, reqLogger) - return reconcile.Result{}, err - } - } - var component render.Component kubeDNSServiceName := utils.GetDNSServiceName(r.opts.DetectedProvider) @@ -374,16 +344,38 @@ func (r *ReconcileWindows) Reconcile(ctx context.Context, request reconcile.Requ return reconcile.Result{}, err } + // Run the variant's windows controller extension to build the render context + // (creating no enterprise artifacts in core). + cc := extensions.ControllerContext{ + RenderContext: extensions.RenderContext{ + Installation: &instance.Spec, + FelixConfiguration: felixConfiguration, + ClusterDomain: r.opts.ClusterDomain, + TrustedBundle: typhaNodeTLS.TrustedBundle, + }, + Controller: extensions.WindowsController, + Ctx: ctx, + Client: r.client, + CertificateManager: certificateManager, + } + if err := r.opts.Extensions.Validate(cc); err != nil { + r.status.SetDegraded(operatorv1.ResourceValidationError, "Invalid installation configuration", err, reqLogger) + return reconcile.Result{}, err + } + renderCtx, _, err := r.opts.Extensions.ExtendContext(cc) + if err != nil { + r.status.SetDegraded(operatorv1.ResourceCreateError, "Error preparing windows extension", err, reqLogger) + return reconcile.Result{}, err + } + windowsCfg := render.WindowsConfiguration{ - K8sServiceEp: k8sapi.Endpoint, - K8sDNSServers: kubeDNSIPs, - Installation: &instance.Spec, - ClusterDomain: r.opts.ClusterDomain, - TLS: typhaNodeTLS, - PrometheusServerTLS: nodePrometheusTLS, - NodeReporterMetricsPort: nodeReporterMetricsPort, - VXLANVNI: *felixConfiguration.Spec.VXLANVNI, - ImageOverrides: r.opts.Extensions.Images(), + K8sServiceEp: k8sapi.Endpoint, + K8sDNSServers: kubeDNSIPs, + Installation: &instance.Spec, + ClusterDomain: r.opts.ClusterDomain, + TLS: typhaNodeTLS, + VXLANVNI: *felixConfiguration.Spec.VXLANVNI, + ImageOverrides: r.opts.Extensions.Images(), } component = render.Windows(&windowsCfg) @@ -409,7 +401,7 @@ func (r *ReconcileWindows) Reconcile(ctx context.Context, request reconcile.Requ r.client, r.scheme, instance, - utils.WithRenderContext(extensions.RenderContext{Installation: &instance.Spec}), + utils.WithRenderContext(renderCtx), utils.WithExtensions(r.opts.Extensions), ) if err := handler.CreateOrUpdateOrDelete(ctx, component, nil); err != nil { diff --git a/pkg/enterprise/installation.go b/pkg/enterprise/installation.go index 4eb89c2282..6b3658f62a 100644 --- a/pkg/enterprise/installation.go +++ b/pkg/enterprise/installation.go @@ -15,7 +15,6 @@ package enterprise import ( - "errors" "fmt" "github.com/tigera/operator/pkg/common" @@ -27,9 +26,9 @@ import ( "github.com/tigera/operator/pkg/tls/certificatemanagement" ) -// controllerExtension is the Calico Enterprise controller-side hook for the +// coreControllerExtension is the Calico Enterprise controller-side hook for the // installation controller. -type controllerExtension struct{} +type coreControllerExtension struct{} // installationRenderData is the controller-produced data the installation // extension hands to its modifiers through RenderContext.Extension. The node @@ -46,20 +45,15 @@ func installationData(rc extensions.RenderContext) installationRenderData { } // Validate rejects installation config Calico Enterprise does not support. -func (controllerExtension) Validate(cc extensions.ControllerContext) error { - // Reject the unsupported zero reporter port. The port value itself is derived - // in the node modifier; only this validation lives here. - if cc.FelixConfiguration.Spec.PrometheusReporterPort != nil && *cc.FelixConfiguration.Spec.PrometheusReporterPort == 0 { - return errors.New("felixConfiguration prometheusReporterPort=0 not supported") - } - return nil +func (coreControllerExtension) Validate(cc extensions.ControllerContext) error { + return validateReporterPort(cc.FelixConfiguration) } // ExtendContext does the controller-side work the modifiers can't: creating and // fetching the certificates that feed the trusted bundle. It returns the render // context carrying the produced node prometheus keypair, and that keypair as one // the controller should manage. -func (controllerExtension) ExtendContext(cc extensions.ControllerContext) (extensions.RenderContext, []certificatemanagement.KeyPairInterface, error) { +func (coreControllerExtension) ExtendContext(cc extensions.ControllerContext) (extensions.RenderContext, []certificatemanagement.KeyPairInterface, error) { rc := cc.RenderContext nodePrometheusTLS, err := cc.CertificateManager.GetOrCreateKeyPair( diff --git a/pkg/enterprise/installation_test.go b/pkg/enterprise/installation_test.go index bd790bc161..2c0f917984 100644 --- a/pkg/enterprise/installation_test.go +++ b/pkg/enterprise/installation_test.go @@ -73,6 +73,7 @@ func newControllerContext(variant operatorv1.ProductVariant) extensions.Controll TrustedBundle: trustedBundle, ClusterDomain: "cluster.local", }, + Controller: extensions.InstallationController, Ctx: context.Background(), Client: c, CertificateManager: certManager, diff --git a/pkg/enterprise/node.go b/pkg/enterprise/node.go index 08f2ecf7f8..ad4469a8b7 100644 --- a/pkg/enterprise/node.go +++ b/pkg/enterprise/node.go @@ -15,6 +15,7 @@ package enterprise import ( + "errors" "fmt" "slices" @@ -263,6 +264,15 @@ func nodeMetricsService(rc extensions.RenderContext) *corev1.Service { } } +// validateReporterPort rejects the unsupported zero prometheus reporter port. +// The node and windows controller extensions share it. +func validateReporterPort(fc *v3.FelixConfiguration) error { + if fc != nil && fc.Spec.PrometheusReporterPort != nil && *fc.Spec.PrometheusReporterPort == 0 { + return errors.New("felixConfiguration prometheusReporterPort=0 not supported") + } + return nil +} + // nodeReporterPort returns the reporter metrics port from the FelixConfiguration, // falling back to the default. The node-metrics Service and the // FELIX_PROMETHEUSREPORTERPORT env var both derive from here so they can't drift. diff --git a/pkg/enterprise/register.go b/pkg/enterprise/register.go index e0d0532b31..4d533a0e33 100644 --- a/pkg/enterprise/register.go +++ b/pkg/enterprise/register.go @@ -28,7 +28,8 @@ func New() *extensions.Set { s := extensions.NewSet() ent := s.Variant(operatorv1.CalicoEnterprise) - ent.Controller(controllerExtension{}) + ent.Controller(extensions.InstallationController, coreControllerExtension{}) + ent.Controller(extensions.WindowsController, windowsControllerExtension{}) registerTypha(ent) registerNode(ent) registerWindows(ent) diff --git a/pkg/enterprise/windows.go b/pkg/enterprise/windows.go index d00e7da88d..bffb7ec1d9 100644 --- a/pkg/enterprise/windows.go +++ b/pkg/enterprise/windows.go @@ -26,9 +26,11 @@ import ( operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/dns" "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" rmeta "github.com/tigera/operator/pkg/render/common/meta" + "github.com/tigera/operator/pkg/tls/certificatemanagement" ) // windowsNodeContainers are the calico-node-windows containers that share the @@ -38,22 +40,61 @@ var windowsNodeContainers = map[string]bool{"felix": true, "node": true, "confd" func registerWindows(v *extensions.Variant) { v.Image(render.ComponentNameWindowsNodeImg, components.ComponentTigeraNodeWindows) v.Image(render.ComponentNameWindowsCNIImg, components.ComponentTigeraCNIWindows) - extensions.RegisterModifier(v, render.ComponentNameWindows, modifyWindows) + v.Modify(render.ComponentNameWindows, modifyWindows) +} + +// windowsControllerExtension is the Calico Enterprise controller-side hook for the +// windows controller. +type windowsControllerExtension struct{} + +// windowsRenderData is the controller-produced data the windows extension hands to +// its modifier through RenderContext.Extension. +type windowsRenderData struct { + prometheusServerTLS certificatemanagement.KeyPairInterface +} + +// windowsData pulls the windows extension's render data back out of the render +// context, returning the zero value when none is set. +func windowsData(rc extensions.RenderContext) windowsRenderData { + data, _ := rc.Extension.(windowsRenderData) + return data +} + +// Validate rejects windows installation config Calico Enterprise does not support. +func (windowsControllerExtension) Validate(cc extensions.ControllerContext) error { + return validateReporterPort(cc.FelixConfiguration) +} + +// ExtendContext fetches the node prometheus keypair the installation controller +// created and stashes it in the render context for the windows modifier. +func (windowsControllerExtension) ExtendContext(cc extensions.ControllerContext) (extensions.RenderContext, []certificatemanagement.KeyPairInterface, error) { + rc := cc.RenderContext + tls, err := cc.CertificateManager.GetKeyPair( + cc.Client, + render.NodePrometheusTLSServerSecret, + common.OperatorNamespace(), + dns.GetServiceDNSNames(render.WindowsNodeMetricsService, common.CalicoNamespace, cc.ClusterDomain), + ) + if err != nil { + return rc, nil, fmt.Errorf("error getting node prometheus TLS certificate: %w", err) + } + rc.Extension = windowsRenderData{prometheusServerTLS: tls} + return rc, nil, nil } // modifyWindows layers Calico Enterprise behavior onto the rendered // calico-node-windows objects: the node-metrics Service and the Enterprise // daemonset configuration (flow/DNS log env, prometheus reporter, trusted DNS // servers, the calico log volume, and the prometheus reporter keypair mount). -func modifyWindows(rc extensions.RenderContext, wc render.WindowsExtensionContext, objs, del []client.Object) ([]client.Object, []client.Object) { +func modifyWindows(rc extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { if ds, ok := extensions.FindObject[*appsv1.DaemonSet](objs, common.WindowsDaemonSetName); ok { - modifyWindowsDaemonSet(rc, wc, ds) + modifyWindowsDaemonSet(rc, ds) } - return append(objs, windowsNodeMetricsService(wc)), del + return append(objs, windowsNodeMetricsService(rc)), del } -func modifyWindowsDaemonSet(rc extensions.RenderContext, wc render.WindowsExtensionContext, ds *appsv1.DaemonSet) { +func modifyWindowsDaemonSet(rc extensions.RenderContext, ds *appsv1.DaemonSet) { dirOrCreate := corev1.HostPathDirectoryOrCreate spec := &ds.Spec.Template.Spec @@ -68,7 +109,7 @@ func modifyWindowsDaemonSet(rc extensions.RenderContext, wc render.WindowsExtens continue } - c.Env = append(c.Env, windowsEnterpriseEnv(rc, wc)...) + c.Env = append(c.Env, windowsEnterpriseEnv(rc)...) // Enterprise mounts the calico log directory in place of the OSS CNI log // directory, so drop the OSS mount before adding the enterprise one. @@ -76,15 +117,16 @@ func modifyWindowsDaemonSet(rc extensions.RenderContext, wc render.WindowsExtens c.VolumeMounts = append(c.VolumeMounts, corev1.VolumeMount{MountPath: "/var/log/calico", Name: "var-log-calico"}) } - mountWindowsPrometheusTLS(wc, ds) + mountWindowsPrometheusTLS(rc, ds) } // windowsEnterpriseEnv is the Enterprise felix configuration added to the // calico-node-windows containers. -func windowsEnterpriseEnv(rc extensions.RenderContext, wc render.WindowsExtensionContext) []corev1.EnvVar { +func windowsEnterpriseEnv(rc extensions.RenderContext) []corev1.EnvVar { + tls := windowsData(rc).prometheusServerTLS env := []corev1.EnvVar{ {Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, - {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: fmt.Sprintf("%d", wc.NodeReporterMetricsPort)}, + {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: fmt.Sprintf("%d", nodeReporterPort(rc.FelixConfiguration))}, {Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, {Name: "FELIX_FLOWLOGSFILEINCLUDELABELS", Value: "true"}, {Name: "FELIX_FLOWLOGSFILEINCLUDEPOLICIES", Value: "true"}, @@ -95,11 +137,11 @@ func windowsEnterpriseEnv(rc extensions.RenderContext, wc render.WindowsExtensio {Name: "FELIX_DNSLOGSFILEPERNODELIMIT", Value: "1000"}, } - if wc.PrometheusServerTLS != nil && wc.TrustedBundle != nil { + if tls != nil && rc.TrustedBundle != nil { env = append(env, - corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERCERTFILE", Value: wc.PrometheusServerTLS.VolumeMountCertificateFilePath()}, - corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERKEYFILE", Value: wc.PrometheusServerTLS.VolumeMountKeyFilePath()}, - corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERCAFILE", Value: wc.TrustedBundle.MountPath()}, + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERCERTFILE", Value: tls.VolumeMountCertificateFilePath()}, + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERKEYFILE", Value: tls.VolumeMountKeyFilePath()}, + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERCAFILE", Value: rc.TrustedBundle.MountPath()}, ) } @@ -117,11 +159,11 @@ func windowsEnterpriseEnv(rc extensions.RenderContext, wc render.WindowsExtensio // mountWindowsPrometheusTLS mounts the node prometheus reporter keypair onto the // windows daemonset: the volume, the volume mount on each node container, and // the pod hash annotation that rolls the pods on cert rotation. -func mountWindowsPrometheusTLS(wc render.WindowsExtensionContext, ds *appsv1.DaemonSet) { - if wc.PrometheusServerTLS == nil { +func mountWindowsPrometheusTLS(rc extensions.RenderContext, ds *appsv1.DaemonSet) { + tls := windowsData(rc).prometheusServerTLS + if tls == nil { return } - tls := wc.PrometheusServerTLS spec := &ds.Spec.Template.Spec spec.Volumes = append(spec.Volumes, tls.Volume()) @@ -141,7 +183,8 @@ func mountWindowsPrometheusTLS(wc render.WindowsExtensionContext, ds *appsv1.Dae // windowsNodeMetricsService builds the enterprise-only calico-node-metrics-windows // Service. -func windowsNodeMetricsService(wc render.WindowsExtensionContext) *corev1.Service { +func windowsNodeMetricsService(rc extensions.RenderContext) *corev1.Service { + reporterPort := nodeReporterPort(rc.FelixConfiguration) return &corev1.Service{ TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{ @@ -155,8 +198,8 @@ func windowsNodeMetricsService(wc render.WindowsExtensionContext) *corev1.Servic Ports: []corev1.ServicePort{ { Name: "calico-metrics-port", - Port: int32(wc.NodeReporterMetricsPort), - TargetPort: intstr.FromInt(wc.NodeReporterMetricsPort), + Port: int32(reporterPort), + TargetPort: intstr.FromInt(reporterPort), Protocol: corev1.ProtocolTCP, }, { diff --git a/pkg/enterprise/windows_test.go b/pkg/enterprise/windows_test.go index d0c11c6642..d4c7967b9b 100644 --- a/pkg/enterprise/windows_test.go +++ b/pkg/enterprise/windows_test.go @@ -15,6 +15,8 @@ package enterprise_test import ( + "context" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -30,9 +32,9 @@ import ( "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/controller/certificatemanager" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/dns" "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" - "github.com/tigera/operator/pkg/tls/certificatemanagement" ) var _ = Describe("windows enterprise image override", func() { @@ -91,16 +93,8 @@ var _ = Describe("windows enterprise modifier", func() { } } - wcFor := func(tls certificatemanagement.KeyPairInterface, bundle certificatemanagement.TrustedBundleRO) render.WindowsExtensionContext { - return render.WindowsExtensionContext{ - NodeReporterMetricsPort: 9081, - PrometheusServerTLS: tls, - TrustedBundle: bundle, - } - } - It("appends the node-metrics service", func() { - out, _ := applyExtensionsWithContext(ext, render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone), wcFor(nil, nil), newObjs(), nil) + out, _ := applyExtensions(ext, render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone), newObjs(), nil) svc, ok := extensions.FindObject[*corev1.Service](out, render.WindowsNodeMetricsService) Expect(ok).To(BeTrue()) Expect(svc.Namespace).To(Equal(common.CalicoNamespace)) @@ -108,7 +102,7 @@ var _ = Describe("windows enterprise modifier", func() { }) It("swaps the cni log mount for the calico log volume and adds enterprise env", func() { - out, _ := applyExtensionsWithContext(ext, render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone), wcFor(nil, nil), newObjs(), nil) + out, _ := applyExtensions(ext, render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone), newObjs(), nil) d := ds(out) Expect(d.Spec.Template.Spec.Volumes).To(ContainElement(HaveField("Name", "var-log-calico"))) @@ -125,7 +119,7 @@ var _ = Describe("windows enterprise modifier", func() { }) It("sets the trusted DNS server on openshift", func() { - out, _ := applyExtensionsWithContext(ext, render.ComponentNameWindows, ctxFor(operatorv1.ProviderOpenShift), wcFor(nil, nil), newObjs(), nil) + out, _ := applyExtensions(ext, render.ComponentNameWindows, ctxFor(operatorv1.ProviderOpenShift), newObjs(), nil) Expect(container(ds(out), "node").Env).To(ContainElement(corev1.EnvVar{Name: "FELIX_DNSTRUSTEDSERVERS", Value: "k8s-service:openshift-dns/dns-default"})) }) @@ -137,9 +131,28 @@ var _ = Describe("windows enterprise modifier", func() { Expect(err).NotTo(HaveOccurred()) tls, err := cm.GetOrCreateKeyPair(cli, render.NodePrometheusTLSServerSecret, common.OperatorNamespace(), []string{"calico-node-metrics-windows"}) Expect(err).NotTo(HaveOccurred()) + // The installation controller persists the secret; do the same here so the + // windows extension's GetKeyPair finds it. + Expect(cli.Create(context.Background(), tls.Secret(common.OperatorNamespace()))).NotTo(HaveOccurred()) bundle := cm.CreateTrustedBundle() - out, _ := applyExtensionsWithContext(ext, render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone), wcFor(tls, bundle), newObjs(), nil) + // Build the render context the way the windows controller does: run the + // windows extension, which fetches the keypair into the context. + cc := extensions.ControllerContext{ + RenderContext: extensions.RenderContext{ + Installation: ctxFor(operatorv1.ProviderNone).Installation, + TrustedBundle: bundle, + ClusterDomain: dns.DefaultClusterDomain, + }, + Controller: extensions.WindowsController, + Ctx: context.Background(), + Client: cli, + CertificateManager: cm, + } + rc, _, err := ext.ExtendContext(cc) + Expect(err).NotTo(HaveOccurred()) + + out, _ := applyExtensions(ext, render.ComponentNameWindows, rc, newObjs(), nil) d := ds(out) Expect(d.Spec.Template.Spec.Volumes).To(ContainElement(tls.Volume())) diff --git a/pkg/extensions/controllerextension.go b/pkg/extensions/controllerextension.go index 05a176bdd9..ce5b713dfd 100644 --- a/pkg/extensions/controllerextension.go +++ b/pkg/extensions/controllerextension.go @@ -23,10 +23,20 @@ import ( "github.com/tigera/operator/pkg/tls/certificatemanagement" ) +// ControllerName identifies the controller a ControllerExtension extends, so a +// variant can register a different hook per controller. Use the constants below +// rather than bare strings so registration and lookup stay in sync. +type ControllerName string + +const ( + InstallationController ControllerName = "installation" + WindowsController ControllerName = "windows" +) + // ControllerExtension extends a controller's reconcile: it validates the // configuration and builds the RenderContext the render phase consumes. The core // operator registers none and runs with the base behavior; an extension build -// registers one. +// registers one per controller it extends. type ControllerExtension interface { // Validate rejects configuration the extension does not support, before any // rendering happens. @@ -47,12 +57,16 @@ type ControllerExtension interface { // deps live here, not on RenderContext, so the modifiers that read RenderContext // can't do I/O - they only transform objects. // -// The controller fills the embedded RenderContext's data fields and the deps; -// ExtendContext does its work, sets the produced artifacts (e.g. -// NodePrometheusTLS) on the embedded context, and returns it. +// Controller names which controller is reconciling, selecting that controller's +// extension hook. The controller fills the embedded RenderContext's data fields, +// the deps, and Controller; ExtendContext does its work, sets the produced +// artifacts on the embedded context, and returns it. type ControllerContext struct { RenderContext + // Controller identifies the reconciling controller, selecting its hook. + Controller ControllerName + Ctx context.Context Client client.Client CertificateManager certificatemanager.CertificateManager diff --git a/pkg/extensions/controllerextension_test.go b/pkg/extensions/controllerextension_test.go index c38d4e5e02..f8808e860a 100644 --- a/pkg/extensions/controllerextension_test.go +++ b/pkg/extensions/controllerextension_test.go @@ -35,6 +35,7 @@ var _ = Describe("controller extension", func() { install := &operatorv1.InstallationSpec{Variant: operatorv1.Calico} rc, _, err := s.ExtendContext(extensions.ControllerContext{ RenderContext: extensions.RenderContext{Installation: install, ClusterDomain: "cluster.local"}, + Controller: extensions.InstallationController, }) Expect(err).NotTo(HaveOccurred()) Expect(rc.Installation).To(BeIdenticalTo(install)) @@ -43,29 +44,30 @@ var _ = Describe("controller extension", func() { }) It("runs the extension registered for the installation variant", func() { - s.Variant(operatorv1.CalicoEnterprise).Controller(fakeController{}) + s.Variant(operatorv1.CalicoEnterprise).Controller(extensions.InstallationController, fakeController{}) rc, _, err := s.ExtendContext(enterpriseContext()) Expect(err).NotTo(HaveOccurred()) Expect(rc.ClusterDomain).To(Equal("from-fake")) }) It("ignores an extension registered for a different variant", func() { - s.Variant(operatorv1.CalicoEnterprise).Controller(fakeController{}) + s.Variant(operatorv1.CalicoEnterprise).Controller(extensions.InstallationController, fakeController{}) rc, _, err := s.ExtendContext(extensions.ControllerContext{ RenderContext: extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}, ClusterDomain: "real"}, + Controller: extensions.InstallationController, }) Expect(err).NotTo(HaveOccurred()) Expect(rc.ClusterDomain).To(Equal("real")) }) It("surfaces the extension error", func() { - s.Variant(operatorv1.CalicoEnterprise).Controller(fakeController{err: errors.New("boom")}) + s.Variant(operatorv1.CalicoEnterprise).Controller(extensions.InstallationController, fakeController{err: errors.New("boom")}) _, _, err := s.ExtendContext(enterpriseContext()) Expect(err).To(MatchError("boom")) }) It("runs the extension's validation", func() { - s.Variant(operatorv1.CalicoEnterprise).Controller(fakeController{validateErr: errors.New("invalid")}) + s.Variant(operatorv1.CalicoEnterprise).Controller(extensions.InstallationController, fakeController{validateErr: errors.New("invalid")}) Expect(s.Validate(enterpriseContext())).To(MatchError("invalid")) }) @@ -83,6 +85,7 @@ var _ = Describe("controller extension", func() { func enterpriseContext() extensions.ControllerContext { return extensions.ControllerContext{ RenderContext: extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}}, + Controller: extensions.InstallationController, } } diff --git a/pkg/extensions/set.go b/pkg/extensions/set.go index ac97f821d5..146d62db6c 100644 --- a/pkg/extensions/set.go +++ b/pkg/extensions/set.go @@ -50,9 +50,10 @@ func NewSet() *Set { func (s *Set) Variant(v operatorv1.ProductVariant) *Variant { if s.variants[v] == nil { s.variants[v] = &Variant{ - variant: v, - modifiers: map[string]decorator{}, - images: s.images, + variant: v, + controllers: map[ControllerName]ControllerExtension{}, + modifiers: map[string]decorator{}, + images: s.images, } } return s.variants[v] @@ -79,7 +80,7 @@ func (s *Set) Decorate(component render.Component, ctx RenderContext) render.Com return s.variant(ctx.Installation.Variant).decorate(component, ctx) } -// Validate runs the controller extension's validation for the installation's +// Validate runs the cc.Controller extension's validation for the installation's // variant, or returns nil when no extension is registered. Nil-safe. func (s *Set) Validate(cc ControllerContext) error { if cc.Installation == nil { @@ -88,9 +89,9 @@ func (s *Set) Validate(cc ControllerContext) error { return s.variant(cc.Installation.Variant).validate(cc) } -// ExtendContext runs the controller extension for the installation's variant and -// returns the resulting RenderContext plus any keypairs the extension wants the -// controller to manage, or the base render context and no keypairs when no +// ExtendContext runs the cc.Controller extension for the installation's variant +// and returns the resulting RenderContext plus any keypairs the extension wants +// the controller to manage, or the base render context and no keypairs when no // extension is registered. Nil-safe. func (s *Set) ExtendContext(cc ControllerContext) (RenderContext, []certificatemanagement.KeyPairInterface, error) { if cc.Installation == nil { diff --git a/pkg/extensions/variant.go b/pkg/extensions/variant.go index 08c920bbc5..ca1c553c4c 100644 --- a/pkg/extensions/variant.go +++ b/pkg/extensions/variant.go @@ -31,20 +31,20 @@ import ( // variant, so within a Variant there is at most one extension per component and // nothing here is itself keyed by variant. type Variant struct { - variant operatorv1.ProductVariant - controller ControllerExtension - modifiers map[string]decorator - images *imageoverride.Overrides // shared with the owning Set + variant operatorv1.ProductVariant + controllers map[ControllerName]ControllerExtension + modifiers map[string]decorator + images *imageoverride.Overrides // shared with the owning Set } // decorator wraps a base component, returning one whose Objects() are augmented // by a registered modifier. type decorator func(base render.Component, rc RenderContext) render.Component -// Controller registers the variant's controller-side extension. A variant has -// at most one; registering replaces any prior one. -func (v *Variant) Controller(c ControllerExtension) { - v.controller = c +// Controller registers the variant's controller-side extension for the named +// controller. A controller has at most one; registering replaces any prior one. +func (v *Variant) Controller(name ControllerName, c ControllerExtension) { + v.controllers[name] = c } // Image registers an image override for the named component. @@ -108,22 +108,22 @@ func (v *Variant) decorate(component render.Component, rc RenderContext) render. return build(component, rc) } -// validate runs the controller extension's validation, or nil when the variant -// has none. Nil-safe. +// validate runs the cc.Controller extension's validation, or nil when the +// variant has none for it. Nil-safe. func (v *Variant) validate(cc ControllerContext) error { - if v == nil || v.controller == nil { + if v == nil || v.controllers[cc.Controller] == nil { return nil } - return v.controller.Validate(cc) + return v.controllers[cc.Controller].Validate(cc) } -// extendContext runs the controller extension, or returns the base render -// context and no managed keypairs when the variant has none. Nil-safe. +// extendContext runs the cc.Controller extension, or returns the base render +// context and no managed keypairs when the variant has none for it. Nil-safe. func (v *Variant) extendContext(cc ControllerContext) (RenderContext, []certificatemanagement.KeyPairInterface, error) { - if v == nil || v.controller == nil { + if v == nil || v.controllers[cc.Controller] == nil { return cc.RenderContext, nil, nil } - return v.controller.ExtendContext(cc) + return v.controllers[cc.Controller].ExtendContext(cc) } // decoratedComponent is the render.Component produced by decorate: it renders diff --git a/pkg/render/node_enterprise_test.go b/pkg/render/node_enterprise_test.go index 9ea5db8d2e..a513ba72d0 100644 --- a/pkg/render/node_enterprise_test.go +++ b/pkg/render/node_enterprise_test.go @@ -102,6 +102,7 @@ var _ = Describe("node enterprise modifier integration", func() { TrustedBundle: typhaNodeTLS.TrustedBundle, ClusterDomain: dns.DefaultClusterDomain, }, + Controller: extensions.InstallationController, Ctx: context.Background(), Client: cli, CertificateManager: certManager, diff --git a/pkg/render/render_test.go b/pkg/render/render_test.go index 242c821bb2..22d7fa63a4 100644 --- a/pkg/render/render_test.go +++ b/pkg/render/render_test.go @@ -66,7 +66,6 @@ func allCalicoComponents( nodeAppArmorProfile string, clusterDomain string, kubeControllersMetricsPort int, - nodeReporterMetricsPort int, bgpLayout *corev1.ConfigMap, logCollector *operatorv1.LogCollector, ) ([]render.Component, error) { @@ -110,13 +109,12 @@ func allCalicoComponents( } winCfg := &render.WindowsConfiguration{ - K8sServiceEp: k8sServiceEp, - K8sDNSServers: []string{}, - Installation: cr, - ClusterDomain: clusterDomain, - TLS: typhaNodeTLS, - NodeReporterMetricsPort: nodeReporterMetricsPort, - VXLANVNI: 4096, + K8sServiceEp: k8sServiceEp, + K8sDNSServers: []string{}, + Installation: cr, + ClusterDomain: clusterDomain, + TLS: typhaNodeTLS, + VXLANVNI: 4096, } nodeCertComponent := rcertificatemanagement.CertificateManagement(&rcertificatemanagement.Config{ @@ -218,7 +216,7 @@ var _ = Describe("Rendering tests", func() { // - 6 kube-controllers resources (ServiceAccount, ClusterRole, Binding, Deployment, Service, Secret,RoleBinding) // - 1 namespace // - 2 Windows node resources (ConfigMap, DaemonSet) - c, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 9094, 0, nil, nil) + c, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 9094, nil, nil) Expect(err).To(BeNil(), "Expected Calico to create successfully %s", err) Expect(componentCount(c)).To(Equal(5 + 3 + 4 + 1 + 6 + 6 + 1 + 2)) }) @@ -231,7 +229,7 @@ var _ = Describe("Rendering tests", func() { var nodeMetricsPort int32 = 9081 instance.Variant = operatorv1.CalicoEnterprise instance.NodeMetricsPort = &nodeMetricsPort - c, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 9094, 0, nil, nil) + c, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 9094, nil, nil) Expect(err).To(BeNil(), "Expected Calico to create successfully %s", err) Expect(componentCount(c)).To(Equal(5 + 3 + 4 + 1 + 6 + 6 + 1 + 2)) }) @@ -244,7 +242,7 @@ var _ = Describe("Rendering tests", func() { instance.Variant = operatorv1.CalicoEnterprise instance.NodeMetricsPort = &nodeMetricsPort - c, err := allCalicoComponents(k8sServiceEp, instance, &operatorv1.ManagementCluster{}, nil, nil, typhaNodeTLS, internalManagerKeyPair, nil, false, "", dns.DefaultClusterDomain, 9094, 0, nil, nil) + c, err := allCalicoComponents(k8sServiceEp, instance, &operatorv1.ManagementCluster{}, nil, nil, typhaNodeTLS, internalManagerKeyPair, nil, false, "", dns.DefaultClusterDomain, 9094, nil, nil) Expect(err).To(BeNil(), "Expected Calico to create successfully %s", err) expectedResources := []client.Object{ @@ -305,7 +303,7 @@ var _ = Describe("Rendering tests", func() { It("should render calico with a apparmor profile if annotation is present in installation", func() { apparmorProf := "foobar" - comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, apparmorProf, dns.DefaultClusterDomain, 0, 0, nil, nil) + comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, apparmorProf, dns.DefaultClusterDomain, 0, nil, nil) Expect(err).To(BeNil(), "Expected Calico to create successfully %s", err) var cn *appsv1.DaemonSet for _, comp := range comps { @@ -329,7 +327,7 @@ var _ = Describe("Rendering tests", func() { } bgpLayout.Name = "bgp-layout" bgpLayout.Namespace = common.OperatorNamespace() - comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 0, 0, bgpLayout, nil) + comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 0, bgpLayout, nil) Expect(err).To(BeNil(), "Expected Calico to create successfully %s", err) var cm *corev1.ConfigMap var ds *appsv1.DaemonSet @@ -354,7 +352,7 @@ var _ = Describe("Rendering tests", func() { testNode := func(processPath operatorv1.CollectProcessPathOption, expectedHostPID bool) { var logCollector operatorv1.LogCollector logCollector.Spec.CollectProcessPath = &processPath - comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 0, 0, nil, &logCollector) + comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 0, nil, &logCollector) Expect(err).To(BeNil(), "Expected Calico to create successfully %s", err) var ds *appsv1.DaemonSet for _, comp := range comps { @@ -385,7 +383,7 @@ var _ = Describe("Rendering tests", func() { }) It("should set node priority class to system-node-critical", func() { - comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 0, 0, nil, nil) + comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 0, nil, nil) Expect(err).To(BeNil(), "Expected Calico to create successfully %s", err) var cn *appsv1.DaemonSet for _, comp := range comps { @@ -401,7 +399,7 @@ var _ = Describe("Rendering tests", func() { }) It("should set typha priority class to system-cluster-critical", func() { - comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 0, 0, nil, nil) + comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 0, nil, nil) Expect(err).To(BeNil(), "Expected Calico to create successfully %s", err) var cn *appsv1.Deployment for _, comp := range comps { @@ -417,7 +415,7 @@ var _ = Describe("Rendering tests", func() { }) It("should set kube controllers priority class to system-cluster-critical", func() { - comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 0, 0, nil, nil) + comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 0, nil, nil) Expect(err).To(BeNil(), "Expected Calico to create successfully %s", err) var cn *appsv1.Deployment for _, comp := range comps { diff --git a/pkg/render/windows.go b/pkg/render/windows.go index 7392094458..dd9692c77f 100644 --- a/pkg/render/windows.go +++ b/pkg/render/windows.go @@ -34,7 +34,6 @@ import ( rcomp "github.com/tigera/operator/pkg/render/common/components" rmeta "github.com/tigera/operator/pkg/render/common/meta" "github.com/tigera/operator/pkg/render/common/securitycontext" - "github.com/tigera/operator/pkg/tls/certificatemanagement" ) const ( @@ -49,14 +48,12 @@ func Windows( } type WindowsConfiguration struct { - K8sServiceEp k8sapi.ServiceEndpoint - K8sDNSServers []string - Installation *operatorv1.InstallationSpec - ClusterDomain string - TLS *TyphaNodeTLS - PrometheusServerTLS certificatemanagement.KeyPairInterface - NodeReporterMetricsPort int - VXLANVNI int + K8sServiceEp k8sapi.ServiceEndpoint + K8sDNSServers []string + Installation *operatorv1.InstallationSpec + ClusterDomain string + TLS *TyphaNodeTLS + VXLANVNI int // ImageOverrides lets a variant swap the windows node and CNI images. The // controller wires in the operator's image overrides; nil resolves to the @@ -99,25 +96,6 @@ func (c *windowsComponent) SupportedOSType() rmeta.OSType { func (c *windowsComponent) ModifierKey() string { return ComponentNameWindows } -// WindowsExtensionContext is the per-component context the windows modifier -// reads (via RenderContext.Component). It carries the enterprise inputs the -// windows controller has but a modifier can't derive from the installation: the -// reporter metrics port, the node prometheus keypair, and the trusted bundle -// the cert env vars reference. -type WindowsExtensionContext struct { - NodeReporterMetricsPort int - PrometheusServerTLS certificatemanagement.KeyPairInterface - TrustedBundle certificatemanagement.TrustedBundleRO -} - -func (c *windowsComponent) ExtensionContext() any { - return WindowsExtensionContext{ - NodeReporterMetricsPort: c.cfg.NodeReporterMetricsPort, - PrometheusServerTLS: c.cfg.PrometheusServerTLS, - TrustedBundle: c.cfg.TLS.TrustedBundle, - } -} - func (c *windowsComponent) Objects() ([]client.Object, []client.Object) { // Clean up old windows upgrader daemonset if present objsToDelete := []client.Object{ diff --git a/pkg/render/windows_test.go b/pkg/render/windows_test.go index 903fb3349b..1d92427a6b 100644 --- a/pkg/render/windows_test.go +++ b/pkg/render/windows_test.go @@ -51,11 +51,7 @@ func renderWindows(cfg *render.WindowsConfiguration) []client.Object { ExpectWithOffset(1, comp.ResolveImages(nil)).To(BeNil()) objs, _ := comp.Objects() rc := extensions.RenderContext{Installation: cfg.Installation} - var extCtx any - if p, ok := comp.(render.ExtensionContextProvider); ok { - extCtx = p.ExtensionContext() - } - out, _ := applyExtensionsWithContext(ext, render.ComponentNameWindows, rc, extCtx, objs, nil) + out, _ := applyExtensions(ext, render.ComponentNameWindows, rc, objs, nil) return out } @@ -718,7 +714,6 @@ var _ = Describe("Windows rendering tests", func() { {name: "calico-node-metrics-windows", ns: "calico-system", group: "", version: "v1", kind: "Service"}, } defaultInstance.Variant = operatorv1.CalicoEnterprise - cfg.NodeReporterMetricsPort = 9081 resources := renderWindows(&cfg) Expect(len(resources)).To(Equal(len(expectedResources))) @@ -1710,7 +1705,6 @@ var _ = Describe("Windows rendering tests", func() { defaultInstance.Variant = operatorv1.CalicoEnterprise defaultInstance.KubernetesProvider = operatorv1.ProviderOpenShift - cfg.NodeReporterMetricsPort = 9081 resources := renderWindows(&cfg) Expect(len(resources)).To(Equal(len(expectedResources))) @@ -1864,7 +1858,6 @@ var _ = Describe("Windows rendering tests", func() { defaultInstance.Variant = operatorv1.CalicoEnterprise defaultInstance.KubernetesProvider = operatorv1.ProviderRKE2 - cfg.NodeReporterMetricsPort = 9081 resources := renderWindows(&cfg) Expect(len(resources)).To(Equal(len(expectedResources)), fmt.Sprintf("Actual resources: %#v", resources)) @@ -2149,7 +2142,6 @@ var _ = Describe("Windows rendering tests", func() { It("should not enable prometheus metrics if NodeMetricsPort is nil", func() { defaultInstance.Variant = operatorv1.CalicoEnterprise defaultInstance.NodeMetricsPort = nil - cfg.NodeReporterMetricsPort = 9081 resources := renderWindows(&cfg) Expect(len(resources)).To(Equal(defaultNumExpectedResources + 1)) From 9a053384379ccad7a7b84b07f5d28a641e332c31 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Thu, 18 Jun 2026 12:51:15 -0700 Subject: [PATCH 33/38] Move the enterprise manager-internal cert fetch into the controller hook The installation hook already fetches the prometheus and esgw certs into the trusted bundle; fold the manager-internal cert in too and drop the IsEnterprise branch from the core controller. --- pkg/controller/installation/core_controller.go | 12 ------------ pkg/enterprise/installation.go | 10 ++++++++++ 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/pkg/controller/installation/core_controller.go b/pkg/controller/installation/core_controller.go index 29fafcccf6..09a55030ea 100644 --- a/pkg/controller/installation/core_controller.go +++ b/pkg/controller/installation/core_controller.go @@ -1086,18 +1086,6 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile return reconcile.Result{}, err } - if instance.Spec.Variant.IsEnterprise() { - managerInternalTLSSecret, err := certificateManager.GetCertificate(r.client, render.ManagerInternalTLSSecretName, common.OperatorNamespace()) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceReadError, fmt.Sprintf("Error fetching TLS secret %s in namespace %s", render.ManagerInternalTLSSecretName, common.OperatorNamespace()), err, reqLogger) - return reconcile.Result{}, nil - } else if managerInternalTLSSecret != nil { - // It may seem odd to add the manager internal TLS secret to the trusted bundle for Typha / calico-node, but this bundle is also used - // for other components in this namespace such as es-kube-controllers, who communicates with Voltron and thus needs to trust this certificate. - typhaNodeTLS.TrustedBundle.AddCertificates(managerInternalTLSSecret) - } - } - birdTemplates, err := getBirdTemplates(r.client) if err != nil { r.status.SetDegraded(operatorv1.ResourceReadError, "Error retrieving confd templates", err, reqLogger) diff --git a/pkg/enterprise/installation.go b/pkg/enterprise/installation.go index 6b3658f62a..934c78d0e7 100644 --- a/pkg/enterprise/installation.go +++ b/pkg/enterprise/installation.go @@ -86,6 +86,16 @@ func (coreControllerExtension) ExtendContext(cc extensions.ControllerContext) (e cc.TrustedBundle.AddCertificates(esgwCertificate) } + // es-kube-controllers talks to Voltron, so the shared bundle must trust the + // manager internal cert. + managerInternalTLS, err := cc.CertificateManager.GetCertificate(cc.Client, render.ManagerInternalTLSSecretName, common.OperatorNamespace()) + if err != nil { + return rc, nil, fmt.Errorf("failed to retrieve %s: %w", render.ManagerInternalTLSSecretName, err) + } + if managerInternalTLS != nil { + cc.TrustedBundle.AddCertificates(managerInternalTLS) + } + var managed []certificatemanagement.KeyPairInterface if nodePrometheusTLS != nil { managed = append(managed, nodePrometheusTLS) From 7f472881942ee128ae3a79fd56480132aec909b2 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Thu, 18 Jun 2026 13:12:20 -0700 Subject: [PATCH 34/38] Let extensions declare their own watches Add a Watcher companion interface to ControllerExtension; each controller's Add() calls Set.SetupWatches, which runs the watch hook of every variant's extension for that controller. The enterprise installation and windows extensions register the enterprise CR and secret watches they need, so core no longer names ManagementCluster, ManagementClusterConnection, LogCollector, or the enterprise prometheus secrets. Still gated by EnterpriseCRDExists for now. --- .../installation/core_controller.go | 23 ++----------------- .../installation/windows_controller.go | 10 ++------ pkg/enterprise/installation.go | 22 ++++++++++++++++++ pkg/enterprise/windows.go | 16 +++++++++++++ pkg/extensions/controllerextension.go | 9 ++++++++ pkg/extensions/controllerextension_test.go | 20 ++++++++++++++++ pkg/extensions/set.go | 21 +++++++++++++++++ 7 files changed, 92 insertions(+), 29 deletions(-) diff --git a/pkg/controller/installation/core_controller.go b/pkg/controller/installation/core_controller.go index 09a55030ea..49121f472f 100644 --- a/pkg/controller/installation/core_controller.go +++ b/pkg/controller/installation/core_controller.go @@ -233,27 +233,8 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { } if opts.EnterpriseCRDExists { - // Watch for changes to primary resource ManagementCluster - err = c.WatchObject(&operatorv1.ManagementCluster{}, &handler.EnqueueRequestForObject{}) - if err != nil { - return fmt.Errorf("tigera-installation-controller failed to watch primary resource: %v", err) - } - - // Watch for changes to primary resource ManagementClusterConnection - err = c.WatchObject(&operatorv1.ManagementClusterConnection{}, &handler.EnqueueRequestForObject{}) - if err != nil { - return fmt.Errorf("tigera-installation-controller failed to watch primary resource: %v", err) - } - - // watch for change to primary resource LogCollector - err = c.WatchObject(&operatorv1.LogCollector{}, &handler.EnqueueRequestForObject{}) - if err != nil { - return fmt.Errorf("tigera-installation-controller failed to watch primary resource: %v", err) - } - - // Watch the internal manager TLS secret in the operator namespace, which included in the bundle for es-kube-controllers. - if err = utils.AddSecretsWatch(c, render.ManagerInternalTLSSecretName, common.OperatorNamespace()); err != nil { - return fmt.Errorf("tigera-installation-controller failed to watch secret: %v", err) + if err = opts.Extensions.SetupWatches(extensions.InstallationController, c); err != nil { + return fmt.Errorf("tigera-installation-controller failed to set up extension watches: %w", err) } if opts.ManageCRDs { diff --git a/pkg/controller/installation/windows_controller.go b/pkg/controller/installation/windows_controller.go index 0e8152d85b..1c05e72227 100644 --- a/pkg/controller/installation/windows_controller.go +++ b/pkg/controller/installation/windows_controller.go @@ -51,7 +51,6 @@ import ( "github.com/tigera/operator/pkg/ctrlruntime" "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" - "github.com/tigera/operator/pkg/render/monitor" ) var logw = logf.Log.WithName("controller_windows") @@ -149,13 +148,8 @@ func AddWindowsController(mgr manager.Manager, opts options.ControllerOptions) e go utils.WaitToAddResourceWatch(c, opts.K8sClientset, logw, ri.ipamConfigWatchReady, []client.Object{&v3.IPAMConfiguration{TypeMeta: metav1.TypeMeta{Kind: v3.KindIPAMConfiguration}}}) if ri.opts.EnterpriseCRDExists { - for _, ns := range []string{common.CalicoNamespace, common.OperatorNamespace()} { - if err = utils.AddSecretsWatch(c, render.NodePrometheusTLSServerSecret, ns); err != nil { - return fmt.Errorf("tigera-windows-controller failed to watch secret '%s' in '%s' namespace: %w", render.NodePrometheusTLSServerSecret, ns, err) - } - if err = utils.AddSecretsWatch(c, monitor.PrometheusClientTLSSecretName, ns); err != nil { - return fmt.Errorf("tigera-windows-controller failed to watch secret '%s' in '%s' namespace: %w", monitor.PrometheusClientTLSSecretName, ns, err) - } + if err = ri.opts.Extensions.SetupWatches(extensions.WindowsController, c); err != nil { + return fmt.Errorf("tigera-windows-controller failed to set up extension watches: %w", err) } } diff --git a/pkg/enterprise/installation.go b/pkg/enterprise/installation.go index 934c78d0e7..58b1360ac5 100644 --- a/pkg/enterprise/installation.go +++ b/pkg/enterprise/installation.go @@ -17,7 +17,13 @@ package enterprise import ( "fmt" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/handler" + + operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/controller/utils" + "github.com/tigera/operator/pkg/ctrlruntime" "github.com/tigera/operator/pkg/dns" "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" @@ -49,6 +55,22 @@ func (coreControllerExtension) Validate(cc extensions.ControllerContext) error { return validateReporterPort(cc.FelixConfiguration) } +// Watches registers the enterprise resources the installation controller +// reconciles on. +func (coreControllerExtension) Watches(c ctrlruntime.Controller) error { + for _, obj := range []client.Object{ + &operatorv1.ManagementCluster{}, + &operatorv1.ManagementClusterConnection{}, + &operatorv1.LogCollector{}, + } { + if err := c.WatchObject(obj, &handler.EnqueueRequestForObject{}); err != nil { + return err + } + } + // es-kube-controllers includes the manager internal TLS secret in its bundle. + return utils.AddSecretsWatch(c, render.ManagerInternalTLSSecretName, common.OperatorNamespace()) +} + // ExtendContext does the controller-side work the modifiers can't: creating and // fetching the certificates that feed the trusted bundle. It returns the render // context carrying the produced node prometheus keypair, and that keypair as one diff --git a/pkg/enterprise/windows.go b/pkg/enterprise/windows.go index bffb7ec1d9..183bfd08ae 100644 --- a/pkg/enterprise/windows.go +++ b/pkg/enterprise/windows.go @@ -26,10 +26,13 @@ import ( operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/controller/utils" + "github.com/tigera/operator/pkg/ctrlruntime" "github.com/tigera/operator/pkg/dns" "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" rmeta "github.com/tigera/operator/pkg/render/common/meta" + "github.com/tigera/operator/pkg/render/monitor" "github.com/tigera/operator/pkg/tls/certificatemanagement" ) @@ -65,6 +68,19 @@ func (windowsControllerExtension) Validate(cc extensions.ControllerContext) erro return validateReporterPort(cc.FelixConfiguration) } +// Watches registers the enterprise secrets the windows controller reconciles on. +func (windowsControllerExtension) Watches(c ctrlruntime.Controller) error { + for _, ns := range []string{common.CalicoNamespace, common.OperatorNamespace()} { + if err := utils.AddSecretsWatch(c, render.NodePrometheusTLSServerSecret, ns); err != nil { + return err + } + if err := utils.AddSecretsWatch(c, monitor.PrometheusClientTLSSecretName, ns); err != nil { + return err + } + } + return nil +} + // ExtendContext fetches the node prometheus keypair the installation controller // created and stashes it in the render context for the windows modifier. func (windowsControllerExtension) ExtendContext(cc extensions.ControllerContext) (extensions.RenderContext, []certificatemanagement.KeyPairInterface, error) { diff --git a/pkg/extensions/controllerextension.go b/pkg/extensions/controllerextension.go index ce5b713dfd..d61b4cee6f 100644 --- a/pkg/extensions/controllerextension.go +++ b/pkg/extensions/controllerextension.go @@ -20,6 +20,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/ctrlruntime" "github.com/tigera/operator/pkg/tls/certificatemanagement" ) @@ -50,6 +51,14 @@ type ControllerExtension interface { ExtendContext(cc ControllerContext) (RenderContext, []certificatemanagement.KeyPairInterface, error) } +// Watcher is an optional companion to ControllerExtension. A controller's Add() +// calls Set.SetupWatches, which invokes Watches on any registered extension that +// implements this, so the extension registers the watches it needs (its CRs, its +// secrets) instead of the controller naming them. +type Watcher interface { + Watches(c ctrlruntime.Controller) error +} + // ControllerContext is the controller-phase context, the corollary to the // render-phase RenderContext. It is the embedded RenderContext (the same data // the render phase sees) plus the controller-side machinery a ControllerExtension diff --git a/pkg/extensions/controllerextension_test.go b/pkg/extensions/controllerextension_test.go index f8808e860a..729e8783d6 100644 --- a/pkg/extensions/controllerextension_test.go +++ b/pkg/extensions/controllerextension_test.go @@ -21,6 +21,7 @@ import ( . "github.com/onsi/gomega" operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/ctrlruntime" "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/tls/certificatemanagement" ) @@ -71,6 +72,13 @@ var _ = Describe("controller extension", func() { Expect(s.Validate(enterpriseContext())).To(MatchError("invalid")) }) + It("runs the watch hook of an extension that implements Watcher", func() { + called := false + s.Variant(operatorv1.CalicoEnterprise).Controller(extensions.InstallationController, watchingController{called: &called}) + Expect(s.SetupWatches(extensions.InstallationController, nil)).NotTo(HaveOccurred()) + Expect(called).To(BeTrue()) + }) + It("returns the base context and no validation error for a nil Set", func() { var nilSet *extensions.Set cc := enterpriseContext() @@ -106,3 +114,15 @@ func (f fakeController) ExtendContext(_ extensions.ControllerContext) (extension } return extensions.RenderContext{ClusterDomain: "from-fake"}, nil, nil } + +// watchingController is a fakeController that also implements the Watcher +// companion, recording that its watch hook ran. +type watchingController struct { + fakeController + called *bool +} + +func (w watchingController) Watches(ctrlruntime.Controller) error { + *w.called = true + return nil +} diff --git a/pkg/extensions/set.go b/pkg/extensions/set.go index 146d62db6c..ee95199a15 100644 --- a/pkg/extensions/set.go +++ b/pkg/extensions/set.go @@ -17,6 +17,7 @@ package extensions import ( operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/ctrlruntime" "github.com/tigera/operator/pkg/imageoverride" "github.com/tigera/operator/pkg/render" "github.com/tigera/operator/pkg/tls/certificatemanagement" @@ -100,6 +101,26 @@ func (s *Set) ExtendContext(cc ControllerContext) (RenderContext, []certificatem return s.variant(cc.Installation.Variant).extendContext(cc) } +// SetupWatches registers the watches every variant's extension declares for the +// named controller. It runs at controller startup, which is variant-agnostic, so +// it registers the union across variants (in practice the one active extension +// build's). Nil-safe. +func (s *Set) SetupWatches(controller ControllerName, c ctrlruntime.Controller) error { + if s == nil { + return nil + } + for _, v := range s.variants { + w, ok := v.controllers[controller].(Watcher) + if !ok { + continue + } + if err := w.Watches(c); err != nil { + return err + } + } + return nil +} + // Images returns the shared image override table. The render package resolves a // component's image through it directly (the imageoverride leaf, so render need // not import extensions). Nil-safe, returning nil overrides that resolve to the From a6e87f94d0bd1044efcc3f08bd277cf88ec54b15 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Thu, 18 Jun 2026 13:24:51 -0700 Subject: [PATCH 35/38] Move LogCollector process-path collection into the node modifier The OSS node render read the enterprise LogCollector CR to set HostPID and FELIX_FLOWLOGSCOLLECTPROCESSPATH - enterprise flow-log behavior (OSS flow logs go through Goldmane, not LogCollector). The installation hook now fetches the LogCollector and records whether process-path collection is on; the node modifier sets HostPID and the env from it. LogCollector is gone from NodeConfiguration and the core controller. --- .../installation/core_controller.go | 10 ---- pkg/enterprise/installation.go | 20 ++++++- pkg/enterprise/node.go | 12 ++++- pkg/render/node.go | 15 ------ pkg/render/node_enterprise_test.go | 37 +++++++++++-- pkg/render/render_test.go | 52 +++---------------- 6 files changed, 70 insertions(+), 76 deletions(-) diff --git a/pkg/controller/installation/core_controller.go b/pkg/controller/installation/core_controller.go index 49121f472f..a52e196046 100644 --- a/pkg/controller/installation/core_controller.go +++ b/pkg/controller/installation/core_controller.go @@ -1007,16 +1007,7 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile var managementCluster *operatorv1.ManagementCluster var managementClusterConnection *operatorv1.ManagementClusterConnection - var logCollector *operatorv1.LogCollector if r.opts.EnterpriseCRDExists { - logCollector, err = utils.GetLogCollector(ctx, r.client) - if logCollector != nil { - if err != nil { - r.status.SetDegraded(operatorv1.ResourceReadError, "Error reading LogCollector", err, reqLogger) - return reconcile.Result{}, err - } - } - managementCluster, err = utils.GetManagementCluster(ctx, r.client) if err != nil { r.status.SetDegraded(operatorv1.ResourceReadError, "Error reading ManagementCluster", err, reqLogger) @@ -1520,7 +1511,6 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile K8sServiceEp: k8sapi.Endpoint, Installation: &instance.Spec, IPPools: crdPoolsToOperator(currentPools.Items), - LogCollector: logCollector, BirdTemplates: birdTemplates, TLS: typhaNodeTLS, ClusterDomain: r.opts.ClusterDomain, diff --git a/pkg/enterprise/installation.go b/pkg/enterprise/installation.go index 58b1360ac5..3126e90880 100644 --- a/pkg/enterprise/installation.go +++ b/pkg/enterprise/installation.go @@ -41,6 +41,10 @@ type coreControllerExtension struct{} // modifier type-asserts it back out. type installationRenderData struct { nodePrometheusTLS certificatemanagement.KeyPairInterface + + // collectProcessPath mirrors LogCollector.Spec.CollectProcessPath being + // enabled; the node modifier uses it to set HostPID and the felix env. + collectProcessPath bool } // installationData pulls the installation extension's render data back out of the @@ -50,6 +54,12 @@ func installationData(rc extensions.RenderContext) installationRenderData { return data } +func collectProcessPathEnabled(lc *operatorv1.LogCollector) bool { + return lc != nil && + lc.Spec.CollectProcessPath != nil && + *lc.Spec.CollectProcessPath == operatorv1.CollectProcessPathEnable +} + // Validate rejects installation config Calico Enterprise does not support. func (coreControllerExtension) Validate(cc extensions.ControllerContext) error { return validateReporterPort(cc.FelixConfiguration) @@ -90,7 +100,15 @@ func (coreControllerExtension) ExtendContext(cc extensions.ControllerContext) (e if nodePrometheusTLS != nil { cc.TrustedBundle.AddCertificates(nodePrometheusTLS) } - rc.Extension = installationRenderData{nodePrometheusTLS: nodePrometheusTLS} + + logCollector, err := utils.GetLogCollector(cc.Ctx, cc.Client) + if err != nil { + return rc, nil, fmt.Errorf("error reading LogCollector: %w", err) + } + rc.Extension = installationRenderData{ + nodePrometheusTLS: nodePrometheusTLS, + collectProcessPath: collectProcessPathEnabled(logCollector), + } prometheusClientCert, err := cc.CertificateManager.GetCertificate(cc.Client, monitor.PrometheusClientTLSSecretName, common.OperatorNamespace()) if err != nil { diff --git a/pkg/enterprise/node.go b/pkg/enterprise/node.go index ad4469a8b7..d4e35dae68 100644 --- a/pkg/enterprise/node.go +++ b/pkg/enterprise/node.go @@ -120,6 +120,11 @@ func nodeEnterpriseRules() []rbacv1.PolicyRule { func modifyNodeDaemonSet(rc extensions.RenderContext, ds *appsv1.DaemonSet) { spec := &ds.Spec.Template.Spec + // Collecting process info for flow logs reads from the host's process table. + if installationData(rc).collectProcessPath { + spec.HostPID = true + } + multiInterfaceMode := multiInterfaceModeEnv(rc.Installation) for i := range spec.InitContainers { @@ -182,7 +187,7 @@ func mountNodePrometheusTLS(rc extensions.RenderContext, ds *appsv1.DaemonSet) { // nodeEnterpriseEnv is the Enterprise felix configuration added to the // calico/node container. func nodeEnterpriseEnv(rc extensions.RenderContext) []corev1.EnvVar { - tls := installationData(rc).nodePrometheusTLS + data := installationData(rc) env := []corev1.EnvVar{ {Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: fmt.Sprintf("%d", nodeReporterPort(rc.FelixConfiguration))}, @@ -196,10 +201,15 @@ func nodeEnterpriseEnv(rc extensions.RenderContext) []corev1.EnvVar { {Name: "FELIX_DNSLOGSFILEPERNODELIMIT", Value: "1000"}, } + if data.collectProcessPath { + env = append(env, corev1.EnvVar{Name: "FELIX_FLOWLOGSCOLLECTPROCESSPATH", Value: "true"}) + } + if mode := multiInterfaceModeEnv(rc.Installation); mode != nil { env = append(env, *mode) } + tls := data.nodePrometheusTLS if tls != nil && rc.TrustedBundle != nil { env = append(env, corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERCERTFILE", Value: tls.VolumeMountCertificateFilePath()}, diff --git a/pkg/render/node.go b/pkg/render/node.go index 4077e43db2..9a838cf5cc 100644 --- a/pkg/render/node.go +++ b/pkg/render/node.go @@ -116,7 +116,6 @@ type NodeConfiguration struct { GoldmaneIP string // Optional fields. - LogCollector *operatorv1.LogCollector MigrateNamespaces bool NodeAppArmorProfile string BirdTemplates map[string]string @@ -1025,10 +1024,6 @@ func (c *nodeComponent) nodeDaemonset(cniCfgMap *corev1.ConfigMap) *appsv1.Daemo ds.Spec.Template.Spec.InitContainers = append(ds.Spec.Template.Spec.InitContainers, c.cniContainer()) } - if c.collectProcessPathEnabled() { - ds.Spec.Template.Spec.HostPID = true - } - setNodeCriticalPod(&(ds.Spec.Template)) if c.cfg.MigrateNamespaces { migration.LimitDaemonSetToMigratedNodes(&ds) @@ -1144,12 +1139,6 @@ func (c *nodeComponent) vppDataplaneEnabled() bool { *c.cfg.Installation.CalicoNetwork.LinuxDataplane == operatorv1.LinuxDataplaneVPP } -func (c *nodeComponent) collectProcessPathEnabled() bool { - return c.cfg.LogCollector != nil && - c.cfg.LogCollector.Spec.CollectProcessPath != nil && - *c.cfg.LogCollector.Spec.CollectProcessPath == operatorv1.CollectProcessPathEnable -} - // cniContainer creates the node's init container that installs CNI. func (c *nodeComponent) cniContainer() corev1.Container { // Determine environment to pass to the CNI init container. @@ -1509,10 +1498,6 @@ func (c *nodeComponent) nodeEnvVars() []corev1.EnvVar { } } - if c.collectProcessPathEnabled() { - nodeEnv = append(nodeEnv, corev1.EnvVar{Name: "FELIX_FLOWLOGSCOLLECTPROCESSPATH", Value: "true"}) - } - // Determine MTU to use. If specified explicitly, use that. Otherwise, set defaults based on an overall // MTU of 1460. mtu := getMTU(c.cfg.Installation) diff --git a/pkg/render/node_enterprise_test.go b/pkg/render/node_enterprise_test.go index a513ba72d0..7696646454 100644 --- a/pkg/render/node_enterprise_test.go +++ b/pkg/render/node_enterprise_test.go @@ -23,6 +23,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -113,7 +114,7 @@ var _ = Describe("node enterprise modifier integration", func() { // renderNodeObjects renders the real node component and applies the registered // modifier, exactly as the componentHandler does. - renderNodeObjects := func() []client.Object { + renderNodeObjects := func(rc extensions.RenderContext) []client.Object { cfg := &render.NodeConfiguration{ K8sServiceEp: k8sapi.ServiceEndpoint{}, Installation: instance, @@ -125,19 +126,19 @@ var _ = Describe("node enterprise modifier integration", func() { comp := render.Node(cfg) Expect(comp.ResolveImages(nil)).NotTo(HaveOccurred()) objs, _ := comp.Objects() - out, _ := applyExtensions(ext, render.ComponentNameNode, renderCtx, objs, nil) + out, _ := applyExtensions(ext, render.ComponentNameNode, rc, objs, nil) return out } It("appends the node metrics service to the real render output", func() { - objs := renderNodeObjects() + objs := renderNodeObjects(renderCtx) svc, ok := extensions.FindObject[*corev1.Service](objs, render.CalicoNodeMetricsService) Expect(ok).To(BeTrue(), "expected the modifier to append %s", render.CalicoNodeMetricsService) Expect(svc.Namespace).To(Equal(common.CalicoNamespace)) }) It("adds the enterprise rules to the real cluster roles", func() { - objs := renderNodeObjects() + objs := renderNodeObjects(renderCtx) nodeRole, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, render.CalicoNodeObjectName) Expect(ok).To(BeTrue()) @@ -149,7 +150,7 @@ var _ = Describe("node enterprise modifier integration", func() { }) It("rewrites the real node daemonset for enterprise", func() { - objs := renderNodeObjects() + objs := renderNodeObjects(renderCtx) ds, ok := extensions.FindObject[*appsv1.DaemonSet](objs, common.NodeDaemonSetName) Expect(ok).To(BeTrue()) @@ -172,6 +173,32 @@ var _ = Describe("node enterprise modifier integration", func() { Expect(c.ReadinessProbe.Exec.Command).To(ContainElement("--bgp-metrics-ready")) }) + It("enables process-path collection when the LogCollector requests it", func() { + enable := operatorv1.CollectProcessPathEnable + Expect(cli.Create(context.Background(), &operatorv1.LogCollector{ + ObjectMeta: metav1.ObjectMeta{Name: "tigera-secure"}, + Spec: operatorv1.LogCollectorSpec{CollectProcessPath: &enable}, + })).NotTo(HaveOccurred()) + + rc, _, err := ext.ExtendContext(extensions.ControllerContext{ + RenderContext: extensions.RenderContext{ + Installation: instance, + TrustedBundle: typhaNodeTLS.TrustedBundle, + ClusterDomain: dns.DefaultClusterDomain, + }, + Controller: extensions.InstallationController, + Ctx: context.Background(), + Client: cli, + CertificateManager: certManager, + }) + Expect(err).NotTo(HaveOccurred()) + + ds, ok := extensions.FindObject[*appsv1.DaemonSet](renderNodeObjects(rc), common.NodeDaemonSetName) + Expect(ok).To(BeTrue()) + Expect(ds.Spec.Template.Spec.HostPID).To(BeTrue()) + Expect(nodeContainer(ds).Env).To(ContainElement(corev1.EnvVar{Name: "FELIX_FLOWLOGSCOLLECTPROCESSPATH", Value: "true"})) + }) + It("adds the enterprise rules to the real typha cluster role", func() { comp := render.Typha(&render.TyphaConfiguration{ K8sServiceEp: k8sapi.ServiceEndpoint{}, diff --git a/pkg/render/render_test.go b/pkg/render/render_test.go index 22d7fa63a4..beef705146 100644 --- a/pkg/render/render_test.go +++ b/pkg/render/render_test.go @@ -67,7 +67,6 @@ func allCalicoComponents( clusterDomain string, kubeControllersMetricsPort int, bgpLayout *corev1.ConfigMap, - logCollector *operatorv1.LogCollector, ) ([]render.Component, error) { namespaces := render.Namespaces(&render.NamespaceConfiguration{Installation: cr, PullSecrets: pullSecrets}) @@ -84,7 +83,6 @@ func allCalicoComponents( NodeAppArmorProfile: nodeAppArmorProfile, ClusterDomain: clusterDomain, BGPLayouts: bgpLayout, - LogCollector: logCollector, BirdTemplates: bt, MigrateNamespaces: up, FelixHealthPort: 9099, @@ -216,7 +214,7 @@ var _ = Describe("Rendering tests", func() { // - 6 kube-controllers resources (ServiceAccount, ClusterRole, Binding, Deployment, Service, Secret,RoleBinding) // - 1 namespace // - 2 Windows node resources (ConfigMap, DaemonSet) - c, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 9094, nil, nil) + c, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 9094, nil) Expect(err).To(BeNil(), "Expected Calico to create successfully %s", err) Expect(componentCount(c)).To(Equal(5 + 3 + 4 + 1 + 6 + 6 + 1 + 2)) }) @@ -229,7 +227,7 @@ var _ = Describe("Rendering tests", func() { var nodeMetricsPort int32 = 9081 instance.Variant = operatorv1.CalicoEnterprise instance.NodeMetricsPort = &nodeMetricsPort - c, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 9094, nil, nil) + c, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 9094, nil) Expect(err).To(BeNil(), "Expected Calico to create successfully %s", err) Expect(componentCount(c)).To(Equal(5 + 3 + 4 + 1 + 6 + 6 + 1 + 2)) }) @@ -242,7 +240,7 @@ var _ = Describe("Rendering tests", func() { instance.Variant = operatorv1.CalicoEnterprise instance.NodeMetricsPort = &nodeMetricsPort - c, err := allCalicoComponents(k8sServiceEp, instance, &operatorv1.ManagementCluster{}, nil, nil, typhaNodeTLS, internalManagerKeyPair, nil, false, "", dns.DefaultClusterDomain, 9094, nil, nil) + c, err := allCalicoComponents(k8sServiceEp, instance, &operatorv1.ManagementCluster{}, nil, nil, typhaNodeTLS, internalManagerKeyPair, nil, false, "", dns.DefaultClusterDomain, 9094, nil) Expect(err).To(BeNil(), "Expected Calico to create successfully %s", err) expectedResources := []client.Object{ @@ -303,7 +301,7 @@ var _ = Describe("Rendering tests", func() { It("should render calico with a apparmor profile if annotation is present in installation", func() { apparmorProf := "foobar" - comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, apparmorProf, dns.DefaultClusterDomain, 0, nil, nil) + comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, apparmorProf, dns.DefaultClusterDomain, 0, nil) Expect(err).To(BeNil(), "Expected Calico to create successfully %s", err) var cn *appsv1.DaemonSet for _, comp := range comps { @@ -327,7 +325,7 @@ var _ = Describe("Rendering tests", func() { } bgpLayout.Name = "bgp-layout" bgpLayout.Namespace = common.OperatorNamespace() - comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 0, bgpLayout, nil) + comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 0, bgpLayout) Expect(err).To(BeNil(), "Expected Calico to create successfully %s", err) var cm *corev1.ConfigMap var ds *appsv1.DaemonSet @@ -348,42 +346,8 @@ var _ = Describe("Rendering tests", func() { Expect(ds.Spec.Template.Annotations["hash.operator.tigera.io/bgp-layout"]).NotTo(BeEmpty()) }) - It("should handle collectProcessPath in logCollector", func() { - testNode := func(processPath operatorv1.CollectProcessPathOption, expectedHostPID bool) { - var logCollector operatorv1.LogCollector - logCollector.Spec.CollectProcessPath = &processPath - comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 0, nil, &logCollector) - Expect(err).To(BeNil(), "Expected Calico to create successfully %s", err) - var ds *appsv1.DaemonSet - for _, comp := range comps { - resources, _ := comp.Objects() - r := rtest.GetResource(resources, "calico-node", "calico-system", "apps", "v1", "DaemonSet") - if r != nil { - ds = r.(*appsv1.DaemonSet) - } - } - checkEnvVar := func(ds *appsv1.DaemonSet) bool { - envPresent := false - for _, env := range ds.Spec.Template.Spec.Containers[0].Env { - if env.Name == "FELIX_FLOWLOGSCOLLECTPROCESSPATH" { - envPresent = true - if env.Value == "true" { - return true - } - } - } - return !envPresent - } - Expect(ds).ToNot(BeNil()) - Expect(ds.Spec.Template.Spec.HostPID).To(Equal(expectedHostPID)) - Expect(checkEnvVar(ds)).To(Equal(true)) - } - testNode(operatorv1.CollectProcessPathEnable, true) - testNode(operatorv1.CollectProcessPathDisable, false) - }) - It("should set node priority class to system-node-critical", func() { - comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 0, nil, nil) + comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 0, nil) Expect(err).To(BeNil(), "Expected Calico to create successfully %s", err) var cn *appsv1.DaemonSet for _, comp := range comps { @@ -399,7 +363,7 @@ var _ = Describe("Rendering tests", func() { }) It("should set typha priority class to system-cluster-critical", func() { - comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 0, nil, nil) + comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 0, nil) Expect(err).To(BeNil(), "Expected Calico to create successfully %s", err) var cn *appsv1.Deployment for _, comp := range comps { @@ -415,7 +379,7 @@ var _ = Describe("Rendering tests", func() { }) It("should set kube controllers priority class to system-cluster-critical", func() { - comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 0, nil, nil) + comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 0, nil) Expect(err).To(BeNil(), "Expected Calico to create successfully %s", err) var cn *appsv1.Deployment for _, comp := range comps { From 632cfa2577a2274899db357deac267e12719d188 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Thu, 18 Jun 2026 14:15:24 -0700 Subject: [PATCH 36/38] Remove variant branches from the kube-controllers render The kube-controllers component renders from a generic config (name, rules, enabled controllers, extra env, network policy) with no IsEnterprise or component-name branching. es-calico-kube-controllers assembly and its constants live in pkg/enterprise and fill that config; calico-kube-controllers still assembles in render. --- pkg/controller/logstorage/common/common.go | 18 +- .../kubecontrollers/es_kube_controllers.go | 10 +- .../es_kube_controllers_test.go | 10 +- pkg/enterprise/kubecontrollers.go | 177 +++++++++ .../kubecontrollers/kube-controllers.go | 363 +++++++----------- .../kubecontrollers/kube-controllers_test.go | 56 ++- 6 files changed, 355 insertions(+), 279 deletions(-) create mode 100644 pkg/enterprise/kubecontrollers.go diff --git a/pkg/controller/logstorage/common/common.go b/pkg/controller/logstorage/common/common.go index 2c070439d7..e15f982b49 100644 --- a/pkg/controller/logstorage/common/common.go +++ b/pkg/controller/logstorage/common/common.go @@ -26,7 +26,7 @@ import ( "github.com/tigera/operator/pkg/controller/utils" "github.com/tigera/operator/pkg/crypto" - "github.com/tigera/operator/pkg/render/kubecontrollers" + "github.com/tigera/operator/pkg/enterprise" ) const ( @@ -45,7 +45,7 @@ const ( // the gateway credentials, and a secret containing real admin level credentials is created and stored in the tigera-elasticsearch namespace to be swapped in once // ES Gateway has confirmed that the gateway credentials match. func CreateKubeControllersSecrets(ctx context.Context, esAdminUserSecret *corev1.Secret, esAdminUserName string, cli client.Client, h utils.NamespaceHelper) (*corev1.Secret, *corev1.Secret, *corev1.Secret, error) { - kubeControllersGatewaySecret, err := utils.GetSecret(ctx, cli, kubecontrollers.ElasticsearchKubeControllersUserSecret, h.TruthNamespace()) + kubeControllersGatewaySecret, err := utils.GetSecret(ctx, cli, enterprise.ElasticsearchKubeControllersUserSecret, h.TruthNamespace()) if err != nil { return nil, nil, nil, err } @@ -53,11 +53,11 @@ func CreateKubeControllersSecrets(ctx context.Context, esAdminUserSecret *corev1 password := crypto.GeneratePassword(16) kubeControllersGatewaySecret = &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ - Name: kubecontrollers.ElasticsearchKubeControllersUserSecret, + Name: enterprise.ElasticsearchKubeControllersUserSecret, Namespace: h.TruthNamespace(), }, Data: map[string][]byte{ - "username": []byte(kubecontrollers.ElasticsearchKubeControllersUserName), + "username": []byte(enterprise.ElasticsearchKubeControllersUserName), "password": []byte(password), }, } @@ -67,34 +67,34 @@ func CreateKubeControllersSecrets(ctx context.Context, esAdminUserSecret *corev1 return nil, nil, nil, err } - kubeControllersVerificationSecret, err := utils.GetSecret(ctx, cli, kubecontrollers.ElasticsearchKubeControllersVerificationUserSecret, h.InstallNamespace()) + kubeControllersVerificationSecret, err := utils.GetSecret(ctx, cli, enterprise.ElasticsearchKubeControllersVerificationUserSecret, h.InstallNamespace()) if err != nil { return nil, nil, nil, err } if kubeControllersVerificationSecret == nil { kubeControllersVerificationSecret = &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ - Name: kubecontrollers.ElasticsearchKubeControllersVerificationUserSecret, + Name: enterprise.ElasticsearchKubeControllersVerificationUserSecret, Namespace: h.InstallNamespace(), Labels: map[string]string{ ESGatewaySelectorLabel: ESGatewaySelectorLabelValue, }, }, Data: map[string][]byte{ - "username": []byte(kubecontrollers.ElasticsearchKubeControllersUserName), + "username": []byte(enterprise.ElasticsearchKubeControllersUserName), "password": hashedPassword, }, } } - kubeControllersSecureUserSecret, err := utils.GetSecret(ctx, cli, kubecontrollers.ElasticsearchKubeControllersSecureUserSecret, h.InstallNamespace()) + kubeControllersSecureUserSecret, err := utils.GetSecret(ctx, cli, enterprise.ElasticsearchKubeControllersSecureUserSecret, h.InstallNamespace()) if err != nil { return nil, nil, nil, err } if kubeControllersSecureUserSecret == nil { kubeControllersSecureUserSecret = &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ - Name: kubecontrollers.ElasticsearchKubeControllersSecureUserSecret, + Name: enterprise.ElasticsearchKubeControllersSecureUserSecret, Namespace: h.InstallNamespace(), Labels: map[string]string{ ESGatewaySelectorLabel: ESGatewaySelectorLabelValue, diff --git a/pkg/controller/logstorage/kubecontrollers/es_kube_controllers.go b/pkg/controller/logstorage/kubecontrollers/es_kube_controllers.go index c51cbc076a..b8f056159e 100644 --- a/pkg/controller/logstorage/kubecontrollers/es_kube_controllers.go +++ b/pkg/controller/logstorage/kubecontrollers/es_kube_controllers.go @@ -44,6 +44,7 @@ import ( "github.com/tigera/operator/pkg/controller/utils" "github.com/tigera/operator/pkg/controller/utils/imageset" "github.com/tigera/operator/pkg/ctrlruntime" + "github.com/tigera/operator/pkg/enterprise" "github.com/tigera/operator/pkg/render" "github.com/tigera/operator/pkg/render/common/networkpolicy" "github.com/tigera/operator/pkg/render/kubecontrollers" @@ -140,7 +141,7 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { if err := utils.AddDeploymentWatch(c, esgateway.DeploymentName, esKubeControllersNamespace.InstallNamespace()); err != nil { return fmt.Errorf("log-storage-access-controller failed to watch the Service resource: %w", err) } - if err := utils.AddDeploymentWatch(c, kubecontrollers.EsKubeController, esKubeControllersNamespace.InstallNamespace()); err != nil { + if err := utils.AddDeploymentWatch(c, enterprise.EsKubeController, esKubeControllersNamespace.InstallNamespace()); err != nil { return fmt.Errorf("log-storage-access-controller failed to watch the Service resource: %w", err) } @@ -168,7 +169,7 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { // Start goroutines to establish watches against projectcalico.org/v3 resources. go utils.WaitToAddTierWatch(networkpolicy.CalicoTierName, c, opts.K8sClientset, log, r.tierWatchReady) go utils.WaitToAddNetworkPolicyWatches(c, opts.K8sClientset, log, []types.NamespacedName{ - {Name: kubecontrollers.EsKubeControllerNetworkPolicyName, Namespace: esKubeControllersNamespace.InstallNamespace()}, + {Name: enterprise.EsKubeControllerNetworkPolicyName, Namespace: esKubeControllersNamespace.InstallNamespace()}, }) return nil @@ -262,7 +263,7 @@ func (r *ESKubeControllersController) Reconcile(ctx context.Context, request rec // Get secrets needed for kube-controllers to talk to elastic. This is needed for zero-tenants and single-tenants // that deploy es-kube-controllers and need to talk to es-gateway var kubeControllersUserSecret *core.Secret - kubeControllersUserSecret, err = utils.GetSecret(ctx, r.client, kubecontrollers.ElasticsearchKubeControllersUserSecret, helper.TruthNamespace()) + kubeControllersUserSecret, err = utils.GetSecret(ctx, r.client, enterprise.ElasticsearchKubeControllersUserSecret, helper.TruthNamespace()) if err != nil { r.status.SetDegraded(operatorv1.ResourceReadError, "Failed to get kube controllers gateway secret", err, reqLogger) return reconcile.Result{}, err @@ -338,13 +339,12 @@ func (r *ESKubeControllersController) Reconcile(ctx context.Context, request rec ClusterDomain: r.clusterDomain, Authentication: authentication, KubeControllersGatewaySecret: kubeControllersUserSecret, - LogStorageExists: logStorage != nil, TrustedBundle: trustedBundle, Namespace: helper.InstallNamespace(), BindingNamespaces: namespaces, Tenant: nil, } - esKubeControllerComponents := kubecontrollers.NewElasticsearchKubeControllers(&kubeControllersCfg) + esKubeControllerComponents := enterprise.NewElasticsearchKubeControllers(&kubeControllersCfg) imageSet, err := imageset.GetImageSet(ctx, r.client, variant) if err != nil { diff --git a/pkg/controller/logstorage/kubecontrollers/es_kube_controllers_test.go b/pkg/controller/logstorage/kubecontrollers/es_kube_controllers_test.go index b35d072664..c053617282 100644 --- a/pkg/controller/logstorage/kubecontrollers/es_kube_controllers_test.go +++ b/pkg/controller/logstorage/kubecontrollers/es_kube_controllers_test.go @@ -46,8 +46,8 @@ import ( "github.com/tigera/operator/pkg/controller/utils" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/enterprise" "github.com/tigera/operator/pkg/render" - "github.com/tigera/operator/pkg/render/kubecontrollers" "github.com/tigera/operator/pkg/render/logstorage" "github.com/tigera/operator/pkg/render/logstorage/esgateway" "github.com/tigera/operator/pkg/tls/certificatemanagement" @@ -235,7 +235,7 @@ var _ = Describe("LogStorage ES kube-controllers controller", func() { dep := appsv1.Deployment{ TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: kubecontrollers.EsKubeController, + Name: enterprise.EsKubeController, Namespace: common.CalicoNamespace, }, } @@ -275,12 +275,12 @@ var _ = Describe("LogStorage ES kube-controllers controller", func() { dep := appsv1.Deployment{ TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: kubecontrollers.EsKubeController, + Name: enterprise.EsKubeController, Namespace: common.CalicoNamespace, }, } Expect(test.GetResource(cli, &dep)).To(BeNil()) - kc := test.GetContainer(dep.Spec.Template.Spec.Containers, kubecontrollers.EsKubeController) + kc := test.GetContainer(dep.Spec.Template.Spec.Containers, enterprise.EsKubeController) Expect(kc).ToNot(BeNil()) Expect(kc.Image).To(Equal(fmt.Sprintf("some.registry.org/%s%s@%s", components.TigeraImagePath, components.ComponentTigeraCalico.Image, "sha256:kubecontrollershash"))) }) @@ -325,7 +325,7 @@ var _ = Describe("LogStorage ES kube-controllers controller", func() { dep := appsv1.Deployment{ TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: kubecontrollers.EsKubeController, + Name: enterprise.EsKubeController, Namespace: common.CalicoNamespace, }, } diff --git a/pkg/enterprise/kubecontrollers.go b/pkg/enterprise/kubecontrollers.go new file mode 100644 index 0000000000..388e7dac4a --- /dev/null +++ b/pkg/enterprise/kubecontrollers.go @@ -0,0 +1,177 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enterprise + +import ( + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + + "github.com/tigera/operator/pkg/render" + relasticsearch "github.com/tigera/operator/pkg/render/common/elasticsearch" + rmeta "github.com/tigera/operator/pkg/render/common/meta" + "github.com/tigera/operator/pkg/render/common/networkpolicy" + "github.com/tigera/operator/pkg/render/kubecontrollers" + "github.com/tigera/operator/pkg/url" +) + +const ( + EsKubeController = "es-calico-kube-controllers" + EsKubeControllerRole = "es-calico-kube-controllers" + EsKubeControllerRoleBinding = "es-calico-kube-controllers" + EsKubeControllerMetrics = "es-calico-kube-controllers-metrics" + EsKubeControllerNetworkPolicyName = networkpolicy.CalicoComponentPolicyPrefix + "es-kube-controller-access" + + ElasticsearchKubeControllersUserSecret = "tigera-ee-kube-controllers-elasticsearch-access" + ElasticsearchKubeControllersUserName = "tigera-ee-kube-controllers" + ElasticsearchKubeControllersSecureUserSecret = "tigera-ee-kube-controllers-elasticsearch-access-gateway" + ElasticsearchKubeControllersVerificationUserSecret = "tigera-ee-kube-controllers-gateway-verification-credentials" +) + +// NewElasticsearchKubeControllers fills the generic kube-controllers configuration +// for the enterprise es-calico-kube-controllers deployment and returns the rendered +// component. es-kube-controllers is a distinct deployment (talks to Elasticsearch via +// es-gateway) reconciled by the logstorage kube-controllers controller, so it's +// assembled here rather than through the render-time modifier mechanism. +func NewElasticsearchKubeControllers(cfg *kubecontrollers.KubeControllersConfiguration) render.Component { + cfg.Name = EsKubeController + cfg.ConfigName = "elasticsearch" + cfg.RoleName = EsKubeControllerRole + cfg.RoleBindingName = EsKubeControllerRoleBinding + cfg.MetricsName = EsKubeControllerMetrics + cfg.DisableConfigAPI = cfg.Tenant.MultiTenant() + + cfg.Rules = kubecontrollers.KubeControllersRoleCommonRules(cfg) + cfg.Rules = append(cfg.Rules, kubecontrollers.KubeControllersRoleEnterpriseCommonRules(cfg)...) + cfg.Rules = append(cfg.Rules, + rbacv1.PolicyRule{ + APIGroups: []string{"elasticsearch.k8s.elastic.co"}, + Resources: []string{"elasticsearches"}, + Verbs: []string{"watch", "get", "list"}, + }, + rbacv1.PolicyRule{ + APIGroups: []string{"rbac.authorization.k8s.io"}, + Resources: []string{"clusterroles", "clusterrolebindings"}, + Verbs: []string{"watch", "list", "get"}, + }, + ) + + if !cfg.Tenant.MultiTenant() { + // Zero and single tenant clusters need elasticsearch configuration. + cfg.EnabledControllers = append(cfg.EnabledControllers, "authorization", "elasticsearchconfiguration") + if cfg.ManagementCluster != nil && cfg.Tenant == nil { + // Enterprise requires the managedcluster controller to push licenses. + cfg.EnabledControllers = append(cfg.EnabledControllers, "managedcluster") + } + } + + cfg.NetworkPolicy = esKubeControllersCalicoSystemPolicy(cfg) + cfg.DeprecatedNetworkPolicyName = "es-kube-controller-access" + cfg.ExtraEnv = esKubeControllersEnv(cfg) + + return kubecontrollers.NewKubeControllers(cfg) +} + +// esKubeControllersEnv builds the enterprise env vars for es-calico-kube-controllers. +func esKubeControllersEnv(cfg *kubecontrollers.KubeControllersConfiguration) []corev1.EnvVar { + var env []corev1.EnvVar + + if cfg.Tenant != nil { + env = append(env, corev1.EnvVar{Name: "TENANT_ID", Value: cfg.Tenant.Spec.ID}) + } + + // What started as a workaround is now the default behaviour. This feature uses our backend in order to + // log into Kibana for users from external identity providers, rather than configuring an authn realm + // in the Elastic stack. + env = append(env, corev1.EnvVar{Name: "ENABLE_ELASTICSEARCH_OIDC_WORKAROUND", Value: "true"}) + if cfg.Authentication != nil { + env = append(env, + corev1.EnvVar{Name: "OIDC_AUTH_USERNAME_PREFIX", Value: cfg.Authentication.Spec.UsernamePrefix}, + corev1.EnvVar{Name: "OIDC_AUTH_GROUP_PREFIX", Value: cfg.Authentication.Spec.GroupsPrefix}, + ) + } + + if cfg.TrustedBundle != nil { + env = append(env, corev1.EnvVar{Name: "MULTI_CLUSTER_FORWARDING_CA", Value: cfg.TrustedBundle.MountPath()}) + } + if cfg.Installation.CalicoNetwork != nil && cfg.Installation.CalicoNetwork.MultiInterfaceMode != nil { + env = append(env, corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: cfg.Installation.CalicoNetwork.MultiInterfaceMode.Value()}) + } + + if !cfg.Tenant.MultiTenant() { + _, esHost, esPort, _ := url.ParseEndpoint(relasticsearch.GatewayEndpoint(rmeta.OSTypeLinux, cfg.ClusterDomain, render.ElasticsearchNamespace)) + env = append(env, + relasticsearch.ElasticHostEnvVar(esHost), + relasticsearch.ElasticPortEnvVar(esPort), + relasticsearch.ElasticUsernameEnvVar(ElasticsearchKubeControllersUserSecret), + relasticsearch.ElasticPasswordEnvVar(ElasticsearchKubeControllersUserSecret), + relasticsearch.ElasticCAEnvVar(rmeta.OSTypeLinux), + ) + } + + return env +} + +func esKubeControllersCalicoSystemPolicy(cfg *kubecontrollers.KubeControllersConfiguration) *v3.NetworkPolicy { + if cfg.ManagementClusterConnection != nil { + return nil + } + + egressRules := []v3.Rule{} + egressRules = networkpolicy.AppendDNSEgressRules(egressRules, cfg.Installation.KubernetesProvider.IsOpenShift()) + egressRules = append(egressRules, []v3.Rule{ + { + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: v3.EntityRule{ + Ports: networkpolicy.Ports(443, 6443, 12388), + }, + }, + }...) + + egressRules = append(egressRules, []v3.Rule{ + { + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: networkpolicy.DefaultHelper().ESGatewayEntityRule(), + }, + }...) + + networkpolicyHelper := networkpolicy.Helper(cfg.Tenant.MultiTenant(), cfg.Namespace) + egressRules = append(egressRules, []v3.Rule{ + { + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: networkpolicyHelper.ManagerEntityRule(), + }, + }...) + + return &v3.NetworkPolicy{ + TypeMeta: metav1.TypeMeta{Kind: "NetworkPolicy", APIVersion: "projectcalico.org/v3"}, + ObjectMeta: metav1.ObjectMeta{ + Name: EsKubeControllerNetworkPolicyName, + Namespace: cfg.Namespace, + }, + Spec: v3.NetworkPolicySpec{ + Order: &networkpolicy.HighPrecedenceOrder, + Tier: networkpolicy.CalicoTierName, + Selector: networkpolicy.KubernetesAppSelector(EsKubeController), + Types: []v3.PolicyType{v3.PolicyTypeEgress}, + Egress: egressRules, + }, + } +} diff --git a/pkg/render/kubecontrollers/kube-controllers.go b/pkg/render/kubecontrollers/kube-controllers.go index 1f592f8358..b6340efb74 100644 --- a/pkg/render/kubecontrollers/kube-controllers.go +++ b/pkg/render/kubecontrollers/kube-controllers.go @@ -38,7 +38,6 @@ import ( "github.com/tigera/operator/pkg/render" "github.com/tigera/operator/pkg/render/applicationlayer" rcomp "github.com/tigera/operator/pkg/render/common/components" - relasticsearch "github.com/tigera/operator/pkg/render/common/elasticsearch" rmeta "github.com/tigera/operator/pkg/render/common/meta" "github.com/tigera/operator/pkg/render/common/networkpolicy" "github.com/tigera/operator/pkg/render/common/secret" @@ -46,7 +45,6 @@ import ( "github.com/tigera/operator/pkg/render/common/securitycontextconstraints" "github.com/tigera/operator/pkg/render/monitor" "github.com/tigera/operator/pkg/tls/certificatemanagement" - "github.com/tigera/operator/pkg/url" ) const ( @@ -72,18 +70,12 @@ const ( // bundle, provisioned by the core controller and passed in as WASMCACert. WASMCACertName = "tigera-waf-ca-bundle" - EsKubeController = "es-calico-kube-controllers" - EsKubeControllerRole = "es-calico-kube-controllers" - EsKubeControllerRoleBinding = "es-calico-kube-controllers" - EsKubeControllerMetrics = "es-calico-kube-controllers-metrics" - EsKubeControllerNetworkPolicyName = networkpolicy.CalicoComponentPolicyPrefix + "es-kube-controller-access" + // ManagedClustersWatchRoleBindingName binds kube-controllers to the managed-cluster + // watch ClusterRole. Used by both calico-kube-controllers (in a management cluster) + // and the enterprise es-calico-kube-controllers, so the binding stays generic here. ManagedClustersWatchRoleBindingName = "es-calico-kube-controllers-managed-cluster-watch" - ElasticsearchKubeControllersUserSecret = "tigera-ee-kube-controllers-elasticsearch-access" - ElasticsearchKubeControllersUserName = "tigera-ee-kube-controllers" - ElasticsearchKubeControllersSecureUserSecret = "tigera-ee-kube-controllers-elasticsearch-access-gateway" - ElasticsearchKubeControllersVerificationUserSecret = "tigera-ee-kube-controllers-gateway-verification-credentials" - KubeControllerPrometheusTLSSecret = "calico-kube-controllers-metrics-tls" + KubeControllerPrometheusTLSSecret = "calico-kube-controllers-metrics-tls" // KubeControllersHealthPort is the port the kube-controllers HealthAggregator listens on when run from the // combined calico binary. The legacy per-component image uses file-based health checks instead. @@ -99,9 +91,6 @@ type KubeControllersConfiguration struct { ManagementClusterConnection *operatorv1.ManagementClusterConnection Authentication *operatorv1.Authentication - // Whether or not the LogStorage CRD is present in the cluster. - LogStorageExists bool - ClusterDomain string MetricsPort int @@ -150,6 +139,39 @@ type KubeControllersConfiguration struct { // caBundle so the apiserver can verify the in-process webhook endpoint. // Only consulted when WAFGatewayExtensionEnabled is true. WAFWebhookCABundle []byte + + // The fields below parameterize the generic kube-controllers component. The + // variant assemblers (NewCalicoKubeControllers, the enterprise es builder) + // fill them; the component renders them without any variant or component-name + // branching. + + // Name is the deployment / pod / container name (and the value the metrics + // Service selects on). + Name string + // ConfigName is the KUBE_CONTROLLERS_CONFIG_NAME the binary reconciles. + ConfigName string + // RoleName / RoleBindingName / MetricsName name the ClusterRole, its binding, + // and the Prometheus metrics Service. + RoleName string + RoleBindingName string + MetricsName string + // EnabledControllers is the ENABLED_CONTROLLERS list. The deployment is only + // rendered when it is non-empty. + EnabledControllers []string + // Rules are the ClusterRole policy rules. + Rules []rbacv1.PolicyRule + // NetworkPolicy, when set, is rendered into the install namespace (and the + // deprecated allow-tigera policy named DeprecatedNetworkPolicyName is deleted). + NetworkPolicy *v3.NetworkPolicy + DeprecatedNetworkPolicyName string + // ExtraEnv is appended to the deployment's container env. + ExtraEnv []corev1.EnvVar + // DisableConfigAPI sets DISABLE_KUBE_CONTROLLERS_CONFIG_API. + DisableConfigAPI bool + // ManageWAFWebhook makes this component own the in-process WAF admission + // webhook surface lifecycle (rendered when WAFGatewayExtensionEnabled, deleted + // otherwise). Only the calico-kube-controllers component sets this. + ManageWAFWebhook bool } func NewCalicoKubeControllersPolicy(cfg *KubeControllersConfiguration, defaultDeny *v3.NetworkPolicy) render.Component { @@ -169,12 +191,27 @@ func NewCalicoKubeControllersPolicy(cfg *KubeControllersConfiguration, defaultDe ) } -func NewCalicoKubeControllers(cfg *KubeControllersConfiguration) *kubeControllersComponent { - kubeControllerRolePolicyRules := kubeControllersRoleCommonRules(cfg) - enabledControllers := []string{"node", "loadbalancer"} +// NewKubeControllers builds a kube-controllers component from a fully-populated +// configuration. Callers (NewCalicoKubeControllers, the enterprise es-kube-controllers +// builder) fill the generic Name/Rules/EnabledControllers/ExtraEnv/NetworkPolicy fields; +// the component renders them with no variant branching. +func NewKubeControllers(cfg *KubeControllersConfiguration) render.Component { + return &kubeControllersComponent{cfg: cfg} +} + +func NewCalicoKubeControllers(cfg *KubeControllersConfiguration) render.Component { + cfg.Name = KubeController + cfg.ConfigName = "default" + cfg.RoleName = KubeControllerRole + cfg.RoleBindingName = KubeControllerRoleBinding + cfg.MetricsName = KubeControllerMetrics + cfg.ManageWAFWebhook = true + + cfg.Rules = KubeControllersRoleCommonRules(cfg) + cfg.EnabledControllers = []string{"node", "loadbalancer"} if cfg.Installation.Variant.IsEnterprise() { - kubeControllerRolePolicyRules = append(kubeControllerRolePolicyRules, kubeControllersRoleEnterpriseCommonRules(cfg)...) - kubeControllerRolePolicyRules = append(kubeControllerRolePolicyRules, + cfg.Rules = append(cfg.Rules, KubeControllersRoleEnterpriseCommonRules(cfg)...) + cfg.Rules = append(cfg.Rules, rbacv1.PolicyRule{ APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, Resources: []string{"remoteclusterconfigurations"}, @@ -196,69 +233,31 @@ func NewCalicoKubeControllers(cfg *KubeControllersConfiguration) *kubeController Verbs: []string{"create", "update", "delete", "watch", "list", "get"}, }, ) - enabledControllers = append(enabledControllers, "service", "federatedservices", "usage") + cfg.EnabledControllers = append(cfg.EnabledControllers, "service", "federatedservices", "usage") if cfg.WAFGatewayExtensionEnabled { - enabledControllers = append(enabledControllers, "applicationlayer") + cfg.EnabledControllers = append(cfg.EnabledControllers, "applicationlayer") } + cfg.ExtraEnv = calicoEnterpriseEnv(cfg) } - return &kubeControllersComponent{ - cfg: cfg, - kubeControllerServiceAccountName: KubeControllerServiceAccount, - kubeControllerRoleName: KubeControllerRole, - kubeControllerRoleBindingName: KubeControllerRoleBinding, - kubeControllerName: KubeController, - kubeControllerConfigName: "default", - kubeControllerMetricsName: KubeControllerMetrics, - kubeControllersRules: kubeControllerRolePolicyRules, - enabledControllers: enabledControllers, - } + return NewKubeControllers(cfg) } -func NewElasticsearchKubeControllers(cfg *KubeControllersConfiguration) *kubeControllersComponent { - var kubeControllerCalicoSystemPolicy *v3.NetworkPolicy - kubeControllerRolePolicyRules := kubeControllersRoleCommonRules(cfg) - - if cfg.Installation.Variant.IsEnterprise() { - kubeControllerRolePolicyRules = append(kubeControllerRolePolicyRules, kubeControllersRoleEnterpriseCommonRules(cfg)...) - kubeControllerRolePolicyRules = append(kubeControllerRolePolicyRules, - rbacv1.PolicyRule{ - APIGroups: []string{"elasticsearch.k8s.elastic.co"}, - Resources: []string{"elasticsearches"}, - Verbs: []string{"watch", "get", "list"}, - }, - rbacv1.PolicyRule{ - APIGroups: []string{"rbac.authorization.k8s.io"}, - Resources: []string{"clusterroles", "clusterrolebindings"}, - Verbs: []string{"watch", "list", "get"}, - }, - ) - - kubeControllerCalicoSystemPolicy = esKubeControllersCalicoSystemPolicy(cfg) +// calicoEnterpriseEnv builds the enterprise-only static env vars for +// calico-kube-controllers. The dynamic WASM_* vars depend on resolved images and +// are added at deployment-render time. +func calicoEnterpriseEnv(cfg *KubeControllersConfiguration) []corev1.EnvVar { + var env []corev1.EnvVar + if cfg.Tenant != nil { + env = append(env, corev1.EnvVar{Name: "TENANT_ID", Value: cfg.Tenant.Spec.ID}) } - - var enabledControllers []string - if !cfg.Tenant.MultiTenant() { - // Zero and single tenant cluster needs elasticsearch configuration - enabledControllers = append(enabledControllers, "authorization", "elasticsearchconfiguration") - if cfg.ManagementCluster != nil && cfg.Tenant == nil { - // Enterprise will require the managedcluster controller to push licenses - enabledControllers = append(enabledControllers, "managedcluster") - } + if cfg.TrustedBundle != nil { + env = append(env, corev1.EnvVar{Name: "MULTI_CLUSTER_FORWARDING_CA", Value: cfg.TrustedBundle.MountPath()}) } - - return &kubeControllersComponent{ - cfg: cfg, - kubeControllerServiceAccountName: KubeControllerServiceAccount, - kubeControllerRoleName: EsKubeControllerRole, - kubeControllerRoleBindingName: EsKubeControllerRoleBinding, - kubeControllerName: EsKubeController, - kubeControllerConfigName: "elasticsearch", - kubeControllerMetricsName: EsKubeControllerMetrics, - kubeControllersRules: kubeControllerRolePolicyRules, - kubeControllerCalicoSystemPolicy: kubeControllerCalicoSystemPolicy, - enabledControllers: enabledControllers, + if cfg.Installation.CalicoNetwork != nil && cfg.Installation.CalicoNetwork.MultiInterfaceMode != nil { + env = append(env, corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: cfg.Installation.CalicoNetwork.MultiInterfaceMode.Value()}) } + return env } type kubeControllersComponent struct { @@ -268,18 +267,6 @@ type kubeControllersComponent struct { // Internal state generated by the given configuration. calicoImage string - kubeControllerServiceAccountName string - kubeControllerRoleName string - kubeControllerRoleBindingName string - kubeControllerName string - kubeControllerConfigName string - kubeControllerMetricsName string - - kubeControllersRules []rbacv1.PolicyRule - kubeControllerCalicoSystemPolicy *v3.NetworkPolicy - - enabledControllers []string - // wasmImage is the fully-resolved OCI reference for the Coraza WAF wasm // binary (Enterprise only). Surfaced to the kube-controllers binary via // the WASM_IMAGE env var; consumed by the applicationlayer reconcilers @@ -296,7 +283,7 @@ func (c *kubeControllersComponent) ResolveImages(is *operatorv1.ImageSet) error if err != nil { return err } - if c.cfg.Installation.Variant.IsEnterprise() && c.cfg.WAFGatewayExtensionEnabled { + if c.cfg.WAFGatewayExtensionEnabled { // The Coraza WAF wasm is baked into the gateway envoy-proxy image as its // final layer; Envoy Gateway extracts it from there. Point WASM_IMAGE at // that same image (no standalone coraza-wasm image needed). @@ -316,12 +303,14 @@ func (c *kubeControllersComponent) Objects() ([]client.Object, []client.Object) objectsToCreate := []client.Object{} objectsToDelete := []client.Object{} - if c.kubeControllerCalicoSystemPolicy != nil { - objectsToCreate = append(objectsToCreate, c.kubeControllerCalicoSystemPolicy) - // allow-tigera Tier was renamed to calico-system - objectsToDelete = append(objectsToDelete, - networkpolicy.DeprecatedAllowTigeraNetworkPolicyObject("es-kube-controller-access", c.cfg.Namespace), - ) + if c.cfg.NetworkPolicy != nil { + objectsToCreate = append(objectsToCreate, c.cfg.NetworkPolicy) + if c.cfg.DeprecatedNetworkPolicyName != "" { + // allow-tigera Tier was renamed to calico-system + objectsToDelete = append(objectsToDelete, + networkpolicy.DeprecatedAllowTigeraNetworkPolicyObject(c.cfg.DeprecatedNetworkPolicyName, c.cfg.Namespace), + ) + } } objectsToCreate = append(objectsToCreate, @@ -331,7 +320,7 @@ func (c *kubeControllersComponent) Objects() ([]client.Object, []client.Object) ) objectsToCreate = append(objectsToCreate, c.managedClusterRoleBindings()...) - if len(c.enabledControllers) > 0 { + if len(c.cfg.EnabledControllers) > 0 { // There's something to run, so create the deployment. objectsToCreate = append(objectsToCreate, c.controllersDeployment()) } else { @@ -358,7 +347,7 @@ func (c *kubeControllersComponent) Objects() ([]client.Object, []client.Object) // ValidatingWebhookConfiguration). Rendered here, rather than as a // passthrough in the core controller, so the objects are cleaned up when the // WAF extension is disabled or the GatewayAPI CR is removed. - if c.kubeControllerName == KubeController { + if c.cfg.ManageWAFWebhook { webhookObjs := applicationlayer.WAFAdmissionWebhookComponents(c.cfg.WAFWebhookCABundle) if c.cfg.WAFGatewayExtensionEnabled { objectsToCreate = append(objectsToCreate, webhookObjs...) @@ -385,7 +374,7 @@ func (c *kubeControllersComponent) Ready() bool { return true } -func kubeControllersRoleCommonRules(cfg *KubeControllersConfiguration) []rbacv1.PolicyRule { +func KubeControllersRoleCommonRules(cfg *KubeControllersConfiguration) []rbacv1.PolicyRule { rules := []rbacv1.PolicyRule{ { // Nodes are watched to monitor for deletions. @@ -511,7 +500,7 @@ func kubeControllersRoleCommonRules(cfg *KubeControllersConfiguration) []rbacv1. return rules } -func kubeControllersRoleEnterpriseCommonRules(cfg *KubeControllersConfiguration) []rbacv1.PolicyRule { +func KubeControllersRoleEnterpriseCommonRules(cfg *KubeControllersConfiguration) []rbacv1.PolicyRule { rules := []rbacv1.PolicyRule{ { APIGroups: []string{""}, @@ -662,7 +651,7 @@ func (c *kubeControllersComponent) controllersServiceAccount() *corev1.ServiceAc return &corev1.ServiceAccount{ TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: c.kubeControllerServiceAccountName, + Name: KubeControllerServiceAccount, Namespace: c.cfg.Namespace, Labels: map[string]string{}, }, @@ -673,9 +662,9 @@ func (c *kubeControllersComponent) controllersClusterRole() *rbacv1.ClusterRole role := &rbacv1.ClusterRole{ TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: c.kubeControllerRoleName, + Name: c.cfg.RoleName, }, - Rules: c.kubeControllersRules, + Rules: c.cfg.Rules, } return role @@ -698,7 +687,7 @@ func (c *kubeControllersComponent) controllersOCPFederationRoleBinding() *rbacv1 Subjects: []rbacv1.Subject{ { Kind: "ServiceAccount", - Name: KubeController, + Name: KubeControllerServiceAccount, Namespace: c.cfg.Namespace, }, }, @@ -707,71 +696,46 @@ func (c *kubeControllersComponent) controllersOCPFederationRoleBinding() *rbacv1 func (c *kubeControllersComponent) controllersDeployment() *appsv1.Deployment { env := []corev1.EnvVar{ - {Name: "KUBE_CONTROLLERS_CONFIG_NAME", Value: c.kubeControllerConfigName}, + {Name: "KUBE_CONTROLLERS_CONFIG_NAME", Value: c.cfg.ConfigName}, {Name: "DATASTORE_TYPE", Value: "kubernetes"}, - {Name: "ENABLED_CONTROLLERS", Value: strings.Join(c.enabledControllers, ",")}, - {Name: "DISABLE_KUBE_CONTROLLERS_CONFIG_API", Value: strconv.FormatBool(c.cfg.Tenant.MultiTenant() && c.kubeControllerConfigName == "elasticsearch")}, + {Name: "ENABLED_CONTROLLERS", Value: strings.Join(c.cfg.EnabledControllers, ",")}, + {Name: "DISABLE_KUBE_CONTROLLERS_CONFIG_API", Value: strconv.FormatBool(c.cfg.DisableConfigAPI)}, } env = append(env, c.cfg.K8sServiceEpPodNetwork.EnvVars()...) - - if c.cfg.Installation.Variant.IsEnterprise() { - if c.cfg.Tenant != nil { - env = append(env, corev1.EnvVar{Name: "TENANT_ID", Value: c.cfg.Tenant.Spec.ID}) - } - - if c.kubeControllerName == EsKubeController { - // What started as a workaround is now the default behaviour. This feature uses our backend in order to - // log into Kibana for users from external identity providers, rather than configuring an authn realm - // in the Elastic stack. - env = append(env, corev1.EnvVar{Name: "ENABLE_ELASTICSEARCH_OIDC_WORKAROUND", Value: "true"}) - - if c.cfg.Authentication != nil { - env = append(env, - corev1.EnvVar{Name: "OIDC_AUTH_USERNAME_PREFIX", Value: c.cfg.Authentication.Spec.UsernamePrefix}, - corev1.EnvVar{Name: "OIDC_AUTH_GROUP_PREFIX", Value: c.cfg.Authentication.Spec.GroupsPrefix}, - ) - } - } - if c.cfg.TrustedBundle != nil { - env = append(env, corev1.EnvVar{Name: "MULTI_CLUSTER_FORWARDING_CA", Value: c.cfg.TrustedBundle.MountPath()}) + env = append(env, c.cfg.ExtraEnv...) + + // Application-layer (gateway-addons / WAF v3) env vars, gated by + // GatewayAPI.spec.extensions.waf.state == Enabled. When the gate is off + // (default), none of the WASM_* env vars are rendered and the kube-controllers + // binary skips the WAF reconcilers entirely (see the applicationlayer entry in + // EnabledControllers). The WASM_IMAGE value depends on resolved images, so this + // is rendered here rather than in the static ExtraEnv. + if c.cfg.WAFGatewayExtensionEnabled { + // Application-layer (gateway-addons) reconcilers consume the Coraza WAF + // wasm OCI reference from this env var to program WAF policy attachments. + // Empty when ResolveImages was not called for the Calico variant; the + // reconciler stamps Programmed=False/WASMUnavailable in that case. + if c.wasmImage != "" { + env = append(env, corev1.EnvVar{Name: "WASM_IMAGE", Value: c.wasmImage}) } - if c.cfg.Installation.CalicoNetwork != nil && c.cfg.Installation.CalicoNetwork.MultiInterfaceMode != nil { - env = append(env, corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: c.cfg.Installation.CalicoNetwork.MultiInterfaceMode.Value()}) + // WASM_PULL_SECRET names the imagePullSecret the reconciler replicates + // from the kube-controllers namespace into a WAFPolicy's namespace so + // the rendered EnvoyExtensionPolicy can pull the wasm OCI artifact from + // a private Tigera registry. Source the name from the first + // Installation.ImagePullSecrets entry so multi-tenant / BYO-registry + // installs reuse whatever pull secret operator already attaches here. + if c.cfg.WASMPullSecret != nil { + env = append(env, corev1.EnvVar{Name: "WASM_PULL_SECRET", Value: c.cfg.WASMPullSecret.Name}) } - // Application-layer (gateway-addons / WAF v3) env vars, gated by - // GatewayAPI.spec.extensions.waf.state == Enabled. When the gate is - // off (default), none of the WASM_* env vars are rendered and the - // kube-controllers binary skips the WAF reconcilers entirely (see the - // applicationlayer entry in enabledControllers). - if c.cfg.WAFGatewayExtensionEnabled { - // Application-layer (gateway-addons) reconcilers consume the Coraza WAF - // wasm OCI reference from this env var to program WAF policy attachments. - // Empty when ResolveImages was not called for the Calico variant; the - // reconciler stamps Programmed=False/WASMUnavailable in that case. - if c.wasmImage != "" { - env = append(env, corev1.EnvVar{Name: "WASM_IMAGE", Value: c.wasmImage}) - } - - // WASM_PULL_SECRET names the imagePullSecret the reconciler replicates - // from the kube-controllers namespace into a WAFPolicy's namespace so - // the rendered EnvoyExtensionPolicy can pull the wasm OCI artifact from - // a private Tigera registry. Source the name from the first - // Installation.ImagePullSecrets entry so multi-tenant / BYO-registry - // installs reuse whatever pull secret operator already attaches here. - if c.cfg.WASMPullSecret != nil { - env = append(env, corev1.EnvVar{Name: "WASM_PULL_SECRET", Value: c.cfg.WASMPullSecret.Name}) - } - - // WASM_CA_CERT names the dedicated CA bundle ConfigMap (provisioned as - // WASMCACert) that the reconciler replicates alongside WASM_PULL_SECRET - // so the EnvoyExtensionPolicy wasm fetcher trusts the registry's TLS - // chain. Only set when the source ConfigMap is actually rendered. - if c.cfg.WASMCACert != nil { - env = append(env, corev1.EnvVar{Name: "WASM_CA_CERT", Value: c.cfg.WASMCACert.Name}) - } + // WASM_CA_CERT names the dedicated CA bundle ConfigMap (provisioned as + // WASMCACert) that the reconciler replicates alongside WASM_PULL_SECRET + // so the EnvoyExtensionPolicy wasm fetcher trusts the registry's TLS + // chain. Only set when the source ConfigMap is actually rendered. + if c.cfg.WASMCACert != nil { + env = append(env, corev1.EnvVar{Name: "WASM_CA_CERT", Value: c.cfg.WASMCACert.Name}) } } @@ -828,7 +792,7 @@ func (c *kubeControllersComponent) controllersDeployment() *appsv1.Deployment { } container := corev1.Container{ - Name: c.kubeControllerName, + Name: c.cfg.Name, Image: c.calicoImage, Command: containerCommand, Env: env, @@ -849,17 +813,6 @@ func (c *kubeControllersComponent) controllersDeployment() *appsv1.Deployment { }) } - if c.kubeControllerName == EsKubeController && !c.cfg.Tenant.MultiTenant() { - _, esHost, esPort, _ := url.ParseEndpoint(relasticsearch.GatewayEndpoint(c.SupportedOSType(), c.cfg.ClusterDomain, render.ElasticsearchNamespace)) - container.Env = append(container.Env, []corev1.EnvVar{ - relasticsearch.ElasticHostEnvVar(esHost), - relasticsearch.ElasticPortEnvVar(esPort), - relasticsearch.ElasticUsernameEnvVar(ElasticsearchKubeControllersUserSecret), - relasticsearch.ElasticPasswordEnvVar(ElasticsearchKubeControllersUserSecret), - relasticsearch.ElasticCAEnvVar(c.SupportedOSType()), - }...) - } - var initContainers []corev1.Container if c.cfg.MetricsServerTLS != nil && c.cfg.MetricsServerTLS.UseCertificateManagement() { initContainers = append(initContainers, c.cfg.MetricsServerTLS.InitContainer(c.cfg.Namespace, sc)) @@ -875,7 +828,7 @@ func (c *kubeControllersComponent) controllersDeployment() *appsv1.Deployment { NodeSelector: c.cfg.Installation.ControlPlaneNodeSelector, Tolerations: tolerations, ImagePullSecrets: c.cfg.Installation.ImagePullSecrets, - ServiceAccountName: c.kubeControllerServiceAccountName, + ServiceAccountName: KubeControllerServiceAccount, InitContainers: initContainers, Containers: []corev1.Container{container}, Volumes: c.kubeControllersVolumes(), @@ -886,7 +839,7 @@ func (c *kubeControllersComponent) controllersDeployment() *appsv1.Deployment { d := appsv1.Deployment{ TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: c.kubeControllerName, + Name: c.cfg.Name, Namespace: c.cfg.Namespace, }, Spec: appsv1.DeploymentSpec{ @@ -896,7 +849,7 @@ func (c *kubeControllersComponent) controllersDeployment() *appsv1.Deployment { }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Name: c.kubeControllerName, + Name: c.cfg.Name, Namespace: c.cfg.Namespace, Annotations: c.annotations(), }, @@ -928,20 +881,20 @@ func (c *kubeControllersComponent) controllersClusterRoleBinding() *rbacv1.Clust for _, ns := range c.cfg.BindingNamespaces { subjects = append(subjects, rbacv1.Subject{ Kind: "ServiceAccount", - Name: c.kubeControllerServiceAccountName, + Name: KubeControllerServiceAccount, Namespace: ns, }) } return &rbacv1.ClusterRoleBinding{ TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: c.kubeControllerRoleBindingName, + Name: c.cfg.RoleBindingName, Labels: map[string]string{}, }, RoleRef: rbacv1.RoleRef{ APIGroup: "rbac.authorization.k8s.io", Kind: "ClusterRole", - Name: c.kubeControllerRoleName, + Name: c.cfg.RoleName, }, Subjects: subjects, } @@ -950,7 +903,7 @@ func (c *kubeControllersComponent) controllersClusterRoleBinding() *rbacv1.Clust func (c *kubeControllersComponent) managedClusterRoleBindings() []client.Object { if c.cfg.ManagementCluster != nil { return []client.Object{ - rcomp.ClusterRoleBinding(ManagedClustersWatchRoleBindingName, render.ManagedClustersWatchClusterRoleName, c.kubeControllerServiceAccountName, []string{c.cfg.Namespace}), + rcomp.ClusterRoleBinding(ManagedClustersWatchRoleBindingName, render.ManagedClustersWatchClusterRoleName, KubeControllerServiceAccount, []string{c.cfg.Namespace}), } } return []client.Object{} @@ -962,16 +915,16 @@ func (c *kubeControllersComponent) prometheusService() *corev1.Service { return &corev1.Service{ TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: c.kubeControllerMetricsName, + Name: c.cfg.MetricsName, Namespace: c.cfg.Namespace, Annotations: map[string]string{ "prometheus.io/scrape": "true", "prometheus.io/port": fmt.Sprintf("%d", c.cfg.MetricsPort), }, - Labels: map[string]string{"k8s-app": c.kubeControllerName}, + Labels: map[string]string{"k8s-app": c.cfg.Name}, }, Spec: corev1.ServiceSpec{ - Selector: map[string]string{"k8s-app": c.kubeControllerName}, + Selector: map[string]string{"k8s-app": c.cfg.Name}, // "Headless" service; prevent kube-proxy from rendering any rules for this service // (which is only intended for Prometheus to scrape). ClusterIP: "None", @@ -1114,53 +1067,3 @@ func kubeControllersCalicoSystemPolicy(cfg *KubeControllersConfiguration) *v3.Ne }, } } - -func esKubeControllersCalicoSystemPolicy(cfg *KubeControllersConfiguration) *v3.NetworkPolicy { - if cfg.ManagementClusterConnection != nil { - return nil - } - - egressRules := []v3.Rule{} - egressRules = networkpolicy.AppendDNSEgressRules(egressRules, cfg.Installation.KubernetesProvider.IsOpenShift()) - egressRules = append(egressRules, []v3.Rule{ - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: v3.EntityRule{ - Ports: networkpolicy.Ports(443, 6443, 12388), - }, - }, - }...) - - egressRules = append(egressRules, []v3.Rule{ - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: networkpolicy.DefaultHelper().ESGatewayEntityRule(), - }, - }...) - - networkpolicyHelper := networkpolicy.Helper(cfg.Tenant.MultiTenant(), cfg.Namespace) - egressRules = append(egressRules, []v3.Rule{ - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: networkpolicyHelper.ManagerEntityRule(), - }, - }...) - - return &v3.NetworkPolicy{ - TypeMeta: metav1.TypeMeta{Kind: "NetworkPolicy", APIVersion: "projectcalico.org/v3"}, - ObjectMeta: metav1.ObjectMeta{ - Name: EsKubeControllerNetworkPolicyName, - Namespace: cfg.Namespace, - }, - Spec: v3.NetworkPolicySpec{ - Order: &networkpolicy.HighPrecedenceOrder, - Tier: networkpolicy.CalicoTierName, - Selector: networkpolicy.KubernetesAppSelector(EsKubeController), - Types: []v3.PolicyType{v3.PolicyTypeEgress}, - Egress: egressRules, - }, - } -} diff --git a/pkg/render/kubecontrollers/kube-controllers_test.go b/pkg/render/kubecontrollers/kube-controllers_test.go index 9f30802d99..8680614827 100644 --- a/pkg/render/kubecontrollers/kube-controllers_test.go +++ b/pkg/render/kubecontrollers/kube-controllers_test.go @@ -41,6 +41,7 @@ import ( "github.com/tigera/operator/pkg/controller/k8sapi" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/enterprise" "github.com/tigera/operator/pkg/render" "github.com/tigera/operator/pkg/render/applicationlayer" rmeta "github.com/tigera/operator/pkg/render/common/meta" @@ -436,23 +437,22 @@ var _ = Describe("kube-controllers rendering tests", func() { version string kind string }{ - {name: kubecontrollers.EsKubeControllerNetworkPolicyName, ns: common.CalicoNamespace, group: "projectcalico.org", version: "v3", kind: "NetworkPolicy"}, + {name: enterprise.EsKubeControllerNetworkPolicyName, ns: common.CalicoNamespace, group: "projectcalico.org", version: "v3", kind: "NetworkPolicy"}, {name: "calico-kube-controllers", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ServiceAccount"}, - {name: kubecontrollers.EsKubeControllerRole, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, - {name: kubecontrollers.EsKubeControllerRoleBinding, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: kubecontrollers.EsKubeController, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "Deployment"}, - {name: kubecontrollers.ElasticsearchKubeControllersUserSecret, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Secret"}, - {name: kubecontrollers.EsKubeControllerMetrics, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Service"}, + {name: enterprise.EsKubeControllerRole, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, + {name: enterprise.EsKubeControllerRoleBinding, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, + {name: enterprise.EsKubeController, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "Deployment"}, + {name: enterprise.ElasticsearchKubeControllersUserSecret, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Secret"}, + {name: enterprise.EsKubeControllerMetrics, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Service"}, } instance.Variant = operatorv1.CalicoEnterprise - cfg.LogStorageExists = true cfg.KubeControllersGatewaySecret = &testutils.KubeControllersUserSecret cfg.MetricsPort = 9094 // Opt in to the WAF Gateway API add-on so the WAF env vars + RBAC are rendered. cfg.WAFGatewayExtensionEnabled = true - component := kubecontrollers.NewElasticsearchKubeControllers(&cfg) + component := enterprise.NewElasticsearchKubeControllers(&cfg) Expect(component.ResolveImages(nil)).To(BeNil()) resources, _ := component.Objects() Expect(len(resources)).To(Equal(len(expectedResources))) @@ -465,7 +465,7 @@ var _ = Describe("kube-controllers rendering tests", func() { } // The Deployment should have the correct configuration. - dp := rtest.GetResource(resources, kubecontrollers.EsKubeController, common.CalicoNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) + dp := rtest.GetResource(resources, enterprise.EsKubeController, common.CalicoNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(dp.Spec.Template.Spec.Containers[0].Image).To(Equal("test-reg/tigera/calico:" + components.ComponentTigeraCalico.Version)) envs := dp.Spec.Template.Spec.Containers[0].Env @@ -482,7 +482,7 @@ var _ = Describe("kube-controllers rendering tests", func() { Expect(dp.Spec.Template.Spec.Volumes[0].Name).To(Equal("tigera-ca-bundle")) Expect(dp.Spec.Template.Spec.Volumes[0].ConfigMap.Name).To(Equal("tigera-ca-bundle")) - clusterRole := rtest.GetResource(resources, kubecontrollers.EsKubeControllerRole, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) + clusterRole := rtest.GetResource(resources, enterprise.EsKubeControllerRole, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) Expect(clusterRole.Rules).To(HaveLen(36), "cluster role should have 36 rules") Expect(clusterRole.Rules).To(ContainElement( rbacv1.PolicyRule{ @@ -693,26 +693,25 @@ var _ = Describe("kube-controllers rendering tests", func() { version string kind string }{ - {name: kubecontrollers.EsKubeControllerNetworkPolicyName, ns: common.CalicoNamespace, group: "projectcalico.org", version: "v3", kind: "NetworkPolicy"}, + {name: enterprise.EsKubeControllerNetworkPolicyName, ns: common.CalicoNamespace, group: "projectcalico.org", version: "v3", kind: "NetworkPolicy"}, {name: "calico-kube-controllers", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ServiceAccount"}, - {name: kubecontrollers.EsKubeControllerRole, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, - {name: kubecontrollers.EsKubeControllerRoleBinding, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, + {name: enterprise.EsKubeControllerRole, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, + {name: enterprise.EsKubeControllerRoleBinding, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, {name: kubecontrollers.ManagedClustersWatchRoleBindingName, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: kubecontrollers.EsKubeController, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "Deployment"}, - {name: kubecontrollers.ElasticsearchKubeControllersUserSecret, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Secret"}, - {name: kubecontrollers.EsKubeControllerMetrics, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Service"}, + {name: enterprise.EsKubeController, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "Deployment"}, + {name: enterprise.ElasticsearchKubeControllersUserSecret, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Secret"}, + {name: enterprise.EsKubeControllerMetrics, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Service"}, } // Override configuration to match expected Enterprise config. instance.Variant = operatorv1.CalicoEnterprise - cfg.LogStorageExists = true cfg.ManagementCluster = &operatorv1.ManagementCluster{} cfg.KubeControllersGatewaySecret = &testutils.KubeControllersUserSecret cfg.MetricsPort = 9094 // Opt in to the WAF Gateway API add-on so the WAF env vars + RBAC are rendered. cfg.WAFGatewayExtensionEnabled = true - component := kubecontrollers.NewElasticsearchKubeControllers(&cfg) + component := enterprise.NewElasticsearchKubeControllers(&cfg) Expect(component.ResolveImages(nil)).To(BeNil()) resources, _ := component.Objects() Expect(len(resources)).To(Equal(len(expectedResources))) @@ -725,7 +724,7 @@ var _ = Describe("kube-controllers rendering tests", func() { } // The Deployment should have the correct configuration. - dp := rtest.GetResource(resources, kubecontrollers.EsKubeController, common.CalicoNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) + dp := rtest.GetResource(resources, enterprise.EsKubeController, common.CalicoNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) envs := dp.Spec.Template.Spec.Containers[0].Env Expect(envs).To(ContainElement(corev1.EnvVar{ @@ -743,7 +742,7 @@ var _ = Describe("kube-controllers rendering tests", func() { Expect(dp.Spec.Template.Spec.Containers[0].Image).To(Equal("test-reg/tigera/calico:" + components.ComponentTigeraCalico.Version)) - clusterRole := rtest.GetResource(resources, kubecontrollers.EsKubeControllerRole, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) + clusterRole := rtest.GetResource(resources, enterprise.EsKubeControllerRole, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) Expect(clusterRole.Rules).To(HaveLen(36), "cluster role should have 36 rules") Expect(clusterRole.Rules).To(ContainElement( rbacv1.PolicyRule{ @@ -865,7 +864,6 @@ var _ = Describe("kube-controllers rendering tests", func() { It("should add the OIDC prefix env variables", func() { instance.Variant = operatorv1.CalicoEnterprise - cfg.LogStorageExists = true cfg.ManagementCluster = &operatorv1.ManagementCluster{} cfg.KubeControllersGatewaySecret = &testutils.KubeControllersUserSecret cfg.MetricsPort = 9094 @@ -875,17 +873,17 @@ var _ = Describe("kube-controllers rendering tests", func() { Openshift: &operatorv1.AuthenticationOpenshift{IssuerURL: "https://api.example.com"}, }} - component := kubecontrollers.NewElasticsearchKubeControllers(&cfg) + component := enterprise.NewElasticsearchKubeControllers(&cfg) Expect(component.ResolveImages(nil)).To(BeNil()) resources, _ := component.Objects() - depResource := rtest.GetResource(resources, kubecontrollers.EsKubeController, common.CalicoNamespace, "apps", "v1", "Deployment") + depResource := rtest.GetResource(resources, enterprise.EsKubeController, common.CalicoNamespace, "apps", "v1", "Deployment") Expect(depResource).ToNot(BeNil()) deployment := depResource.(*appsv1.Deployment) var usernamePrefix, groupPrefix string for _, container := range deployment.Spec.Template.Spec.Containers { - if container.Name == kubecontrollers.EsKubeController { + if container.Name == enterprise.EsKubeController { for _, env := range container.Env { switch env.Name { case "OIDC_AUTH_USERNAME_PREFIX": @@ -1132,20 +1130,19 @@ var _ = Describe("kube-controllers rendering tests", func() { When("enableESOIDCWorkaround is true", func() { It("should set the ENABLE_ELASTICSEARCH_OIDC_WORKAROUND env variable to true", func() { instance.Variant = operatorv1.CalicoEnterprise - cfg.LogStorageExists = true cfg.ManagementCluster = &operatorv1.ManagementCluster{} cfg.KubeControllersGatewaySecret = &testutils.KubeControllersUserSecret cfg.MetricsPort = 9094 - component := kubecontrollers.NewElasticsearchKubeControllers(&cfg) + component := enterprise.NewElasticsearchKubeControllers(&cfg) resources, _ := component.Objects() - depResource := rtest.GetResource(resources, kubecontrollers.EsKubeController, common.CalicoNamespace, "apps", "v1", "Deployment") + depResource := rtest.GetResource(resources, enterprise.EsKubeController, common.CalicoNamespace, "apps", "v1", "Deployment") Expect(depResource).ToNot(BeNil()) deployment := depResource.(*appsv1.Deployment) var esLicenseType string for _, container := range deployment.Spec.Template.Spec.Containers { - if container.Name == kubecontrollers.EsKubeController { + if container.Name == enterprise.EsKubeController { for _, env := range container.Env { if env.Name == "ENABLE_ELASTICSEARCH_OIDC_WORKAROUND" { esLicenseType = env.Value @@ -1289,10 +1286,9 @@ var _ = Describe("kube-controllers rendering tests", func() { cfg.ManagementClusterConnection = nil } instance.Variant = operatorv1.CalicoEnterprise - cfg.LogStorageExists = true cfg.KubeControllersGatewaySecret = &testutils.KubeControllersUserSecret - component := kubecontrollers.NewElasticsearchKubeControllers(&cfg) + component := enterprise.NewElasticsearchKubeControllers(&cfg) resources, _ := component.Objects() policy := testutils.GetCalicoSystemPolicyFromResources(policyName, resources) From 698254f6190979cf2c1d29e7f197974e8758fbbe Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Thu, 18 Jun 2026 14:33:23 -0700 Subject: [PATCH 37/38] Move calico-kube-controllers metrics TLS into the enterprise extension The installation extension hook creates the kube-controllers metrics serving keypair and returns it as a managed keypair; a calico-kube-controllers modifier mounts it onto the deployment (env, volume, mount, cert-management init container, hash annotation). The kube-controllers render base no longer carries MetricsServerTLS, and the installation controller no longer creates that certificate. The component reports a config-driven modifier key so the shared es-calico-kube-controllers deployment, which leaves it empty, is never decorated. --- .../installation/core_controller.go | 28 ---- pkg/controller/utils/component.go | 2 +- pkg/enterprise/installation.go | 25 ++++ pkg/enterprise/installation_test.go | 11 +- pkg/enterprise/kubecontrollers.go | 55 ++++++++ pkg/enterprise/kubecontrollers_test.go | 124 ++++++++++++++++++ pkg/enterprise/register.go | 1 + pkg/render/component.go | 5 + .../kubecontrollers/kube-controllers.go | 35 ++--- .../kubecontrollers/kube-controllers_test.go | 60 +-------- 10 files changed, 236 insertions(+), 110 deletions(-) create mode 100644 pkg/enterprise/kubecontrollers_test.go diff --git a/pkg/controller/installation/core_controller.go b/pkg/controller/installation/core_controller.go index a52e196046..6c5686fee8 100644 --- a/pkg/controller/installation/core_controller.go +++ b/pkg/controller/installation/core_controller.go @@ -85,7 +85,6 @@ import ( "github.com/tigera/operator/pkg/render/common/resourcequota" "github.com/tigera/operator/pkg/render/goldmane" "github.com/tigera/operator/pkg/render/kubecontrollers" - "github.com/tigera/operator/pkg/render/monitor" "github.com/tigera/operator/pkg/tls/certificatemanagement" ) @@ -1191,30 +1190,6 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile return reconcile.Result{}, err } - // Secure calico kube controller metrics. - var kubeControllerTLS certificatemanagement.KeyPairInterface - if instance.Spec.Variant.IsEnterprise() { - // Create or Get TLS certificates for kube controller. - kubeControllerTLS, err = certificateManager.GetOrCreateKeyPair( - r.client, - kubecontrollers.KubeControllerPrometheusTLSSecret, - common.OperatorNamespace(), - dns.GetServiceDNSNames(kubecontrollers.KubeControllerMetrics, common.CalicoNamespace, r.opts.ClusterDomain)) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceReadError, "Error finding or creating TLS certificate kube controllers metric", err, reqLogger) - return reconcile.Result{}, err - } - - // Add prometheus client certificate to Trusted bundle. - kubeControllerPrometheusTLS, err := certificateManager.GetCertificate(r.client, monitor.PrometheusClientTLSSecretName, common.OperatorNamespace()) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceReadError, "Failed to get certificate for kube controllers", err, reqLogger) - return reconcile.Result{}, err - } else if kubeControllerPrometheusTLS != nil { - typhaNodeTLS.TrustedBundle.AddCertificates(kubeControllerTLS, kubeControllerPrometheusTLS) - } - } - nodeAppArmorProfile := "" a := instance.GetObjectMeta().GetAnnotations() if val, ok := a[techPreviewFeatureSeccompApparmor]; ok { @@ -1324,7 +1299,6 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile rcertificatemanagement.NewKeyPairOption(typhaNodeTLS.NodeSecret, true, true), rcertificatemanagement.NewKeyPairOption(typhaNodeTLS.TyphaSecret, true, true), rcertificatemanagement.NewKeyPairOption(typhaNodeTLS.TyphaSecretNonClusterHost, true, true), - rcertificatemanagement.NewKeyPairOption(kubeControllerTLS, true, true), // Nil when the WAF v3 surface is disabled; the certificate-management // render skips nil key pairs. rcertificatemanagement.NewKeyPairOption(wafWebhookTLS, true, true), @@ -1604,7 +1578,6 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile ClusterDomain: r.opts.ClusterDomain, MetricsPort: kubeControllersMetricsPort, Terminating: installationMarkedForDeletion, - MetricsServerTLS: kubeControllerTLS, TrustedBundle: typhaNodeTLS.TrustedBundle, Namespace: common.CalicoNamespace, BindingNamespaces: []string{common.CalicoNamespace}, @@ -1756,7 +1729,6 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile render.TyphaTLSSecretName: typhaNodeTLS.TyphaSecret, render.NodeTLSSecretName: typhaNodeTLS.NodeSecret, render.TyphaTLSSecretName + render.TyphaNonClusterHostSuffix: typhaNodeTLS.TyphaSecretNonClusterHost, - kubecontrollers.KubeControllerPrometheusTLSSecret: kubeControllerTLS, } for _, kp := range managedKeyPairs { keyPairWarnings[kp.GetName()] = kp diff --git a/pkg/controller/utils/component.go b/pkg/controller/utils/component.go index 52eb2dc374..b8f97b1fc3 100644 --- a/pkg/controller/utils/component.go +++ b/pkg/controller/utils/component.go @@ -458,7 +458,7 @@ func resetMetadataForCreate(obj client.Object) { } func (c *componentHandler) CreateOrUpdateOrDelete(ctx context.Context, component render.Component, status status.StatusManager) error { - if ext, ok := component.(render.Extensible); ok && c.extensions == nil { + if ext, ok := component.(render.Extensible); ok && ext.ModifierKey() != "" && c.extensions == nil { c.log.Info("BUG: extensible component rendered by a handler with no extension Set; extensions will not be applied", "component", ext.ModifierKey()) } component = c.extensions.Decorate(component, c.renderCtx) diff --git a/pkg/enterprise/installation.go b/pkg/enterprise/installation.go index 3126e90880..12eef67b13 100644 --- a/pkg/enterprise/installation.go +++ b/pkg/enterprise/installation.go @@ -28,6 +28,7 @@ import ( "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" relasticsearch "github.com/tigera/operator/pkg/render/common/elasticsearch" + "github.com/tigera/operator/pkg/render/kubecontrollers" "github.com/tigera/operator/pkg/render/monitor" "github.com/tigera/operator/pkg/tls/certificatemanagement" ) @@ -42,6 +43,10 @@ type coreControllerExtension struct{} type installationRenderData struct { nodePrometheusTLS certificatemanagement.KeyPairInterface + // kubeControllerTLS is the calico-kube-controllers metrics serving keypair; the + // kube-controllers modifier mounts it onto the deployment. + kubeControllerTLS certificatemanagement.KeyPairInterface + // collectProcessPath mirrors LogCollector.Spec.CollectProcessPath being // enabled; the node modifier uses it to set HostPID and the felix env. collectProcessPath bool @@ -101,12 +106,29 @@ func (coreControllerExtension) ExtendContext(cc extensions.ControllerContext) (e cc.TrustedBundle.AddCertificates(nodePrometheusTLS) } + // The calico-kube-controllers metrics endpoint is served with mTLS in + // Enterprise; the keypair is created here (cluster side effect) and mounted by + // the kube-controllers modifier. + kubeControllerTLS, err := cc.CertificateManager.GetOrCreateKeyPair( + cc.Client, + kubecontrollers.KubeControllerPrometheusTLSSecret, + common.OperatorNamespace(), + dns.GetServiceDNSNames(kubecontrollers.KubeControllerMetrics, common.CalicoNamespace, cc.ClusterDomain), + ) + if err != nil { + return rc, nil, fmt.Errorf("error creating kube-controllers metrics TLS certificate: %w", err) + } + if kubeControllerTLS != nil { + cc.TrustedBundle.AddCertificates(kubeControllerTLS) + } + logCollector, err := utils.GetLogCollector(cc.Ctx, cc.Client) if err != nil { return rc, nil, fmt.Errorf("error reading LogCollector: %w", err) } rc.Extension = installationRenderData{ nodePrometheusTLS: nodePrometheusTLS, + kubeControllerTLS: kubeControllerTLS, collectProcessPath: collectProcessPathEnabled(logCollector), } @@ -140,5 +162,8 @@ func (coreControllerExtension) ExtendContext(cc extensions.ControllerContext) (e if nodePrometheusTLS != nil { managed = append(managed, nodePrometheusTLS) } + if kubeControllerTLS != nil { + managed = append(managed, kubeControllerTLS) + } return rc, managed, nil } diff --git a/pkg/enterprise/installation_test.go b/pkg/enterprise/installation_test.go index 2c0f917984..d99e18eb00 100644 --- a/pkg/enterprise/installation_test.go +++ b/pkg/enterprise/installation_test.go @@ -30,10 +30,10 @@ import ( ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/render/kubecontrollers" ) var _ = Describe("installation controller extension", func() { - It("rejects a zero prometheus reporter port", func() { port := 0 cc := newControllerContext(operatorv1.CalicoEnterprise) @@ -43,11 +43,14 @@ var _ = Describe("installation controller extension", func() { Expect(ext.Validate(cc)).To(HaveOccurred()) }) - It("creates the node prometheus keypair for the enterprise variant", func() { + It("manages the node prometheus and kube-controllers metrics keypairs for the enterprise variant", func() { _, managed, err := ext.ExtendContext(newControllerContext(operatorv1.CalicoEnterprise)) Expect(err).NotTo(HaveOccurred()) - Expect(managed).To(HaveLen(1), "expected the node prometheus keypair to be managed") - Expect(managed[0].GetName()).To(Equal(render.NodePrometheusTLSServerSecret)) + names := []string{} + for _, kp := range managed { + names = append(names, kp.GetName()) + } + Expect(names).To(ConsistOf(render.NodePrometheusTLSServerSecret, kubecontrollers.KubeControllerPrometheusTLSSecret)) }) It("is a no-op for the Calico variant", func() { diff --git a/pkg/enterprise/kubecontrollers.go b/pkg/enterprise/kubecontrollers.go index 388e7dac4a..884f928750 100644 --- a/pkg/enterprise/kubecontrollers.go +++ b/pkg/enterprise/kubecontrollers.go @@ -15,20 +15,75 @@ package enterprise import ( + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" relasticsearch "github.com/tigera/operator/pkg/render/common/elasticsearch" rmeta "github.com/tigera/operator/pkg/render/common/meta" "github.com/tigera/operator/pkg/render/common/networkpolicy" "github.com/tigera/operator/pkg/render/kubecontrollers" + "github.com/tigera/operator/pkg/render/monitor" "github.com/tigera/operator/pkg/url" ) +// registerKubeControllers registers the calico-kube-controllers modifier. There is +// no image override: kube-controllers runs from the combined calico image, which +// resolves by variant in the base render. +func registerKubeControllers(v *extensions.Variant) { + v.Modify(render.ComponentNameKubeControllers, modifyKubeControllers) +} + +// modifyKubeControllers layers the Calico Enterprise metrics serving TLS onto the +// rendered calico-kube-controllers deployment: the env pointing at the keypair, the +// volume + mount, the cert-management init container (when in use), and the pod hash +// annotation that rolls the pod on cert rotation. The keypair has cluster side +// effects, so the installation extension creates it and hands it in via rc. In core +// (calico) it is never created, so the base deployment carries no metrics TLS. +func modifyKubeControllers(rc extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { + tls := installationData(rc).kubeControllerTLS + if tls == nil { + return objs, del + } + + dp, ok := extensions.FindObject[*appsv1.Deployment](objs, kubecontrollers.KubeController) + if !ok { + return objs, del + } + spec := &dp.Spec.Template.Spec + spec.Volumes = append(spec.Volumes, tls.Volume()) + + for i := range spec.Containers { + c := &spec.Containers[i] + if c.Name != kubecontrollers.KubeController { + continue + } + c.Env = append(c.Env, + corev1.EnvVar{Name: "TLS_KEY_PATH", Value: tls.VolumeMountKeyFilePath()}, + corev1.EnvVar{Name: "TLS_CRT_PATH", Value: tls.VolumeMountCertificateFilePath()}, + corev1.EnvVar{Name: "CLIENT_COMMON_NAME", Value: monitor.PrometheusClientTLSSecretName}, + ) + c.VolumeMounts = append(c.VolumeMounts, tls.VolumeMount(rmeta.OSTypeLinux)) + if tls.UseCertificateManagement() { + spec.InitContainers = append(spec.InitContainers, tls.InitContainer(common.CalicoNamespace, c.SecurityContext)) + } + } + + if dp.Spec.Template.Annotations == nil { + dp.Spec.Template.Annotations = map[string]string{} + } + dp.Spec.Template.Annotations[tls.HashAnnotationKey()] = tls.HashAnnotationValue() + + return objs, del +} + const ( EsKubeController = "es-calico-kube-controllers" EsKubeControllerRole = "es-calico-kube-controllers" diff --git a/pkg/enterprise/kubecontrollers_test.go b/pkg/enterprise/kubecontrollers_test.go new file mode 100644 index 0000000000..41ee40ca72 --- /dev/null +++ b/pkg/enterprise/kubecontrollers_test.go @@ -0,0 +1,124 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enterprise_test + +import ( + "context" + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/apis" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/controller/certificatemanager" + ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/render" + rmeta "github.com/tigera/operator/pkg/render/common/meta" + "github.com/tigera/operator/pkg/render/kubecontrollers" + "github.com/tigera/operator/pkg/render/monitor" + "github.com/tigera/operator/pkg/tls" +) + +var _ = Describe("kube-controllers enterprise modifier", func() { + // kubeControllersDeployment is a minimal stand-in for the calico-kube-controllers + // deployment the base render produces, so the modifier has something to mount onto. + kubeControllersDeployment := func() *appsv1.Deployment { + return &appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: kubecontrollers.KubeController, Namespace: common.CalicoNamespace}, + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: kubecontrollers.KubeController}}, + }, + }, + }, + } + } + + It("mounts the metrics serving TLS keypair onto the deployment", func() { + rc, _, err := ext.ExtendContext(newControllerContext(operatorv1.CalicoEnterprise)) + Expect(err).NotTo(HaveOccurred()) + + objs, _ := applyExtensions(ext, render.ComponentNameKubeControllers, rc, []client.Object{kubeControllersDeployment()}, nil) + dp, ok := extensions.FindObject[*appsv1.Deployment](objs, kubecontrollers.KubeController) + Expect(ok).To(BeTrue()) + + c := dp.Spec.Template.Spec.Containers[0] + Expect(c.Env).To(ContainElements( + corev1.EnvVar{Name: "TLS_KEY_PATH", Value: "/calico-kube-controllers-metrics-tls/tls.key"}, + corev1.EnvVar{Name: "TLS_CRT_PATH", Value: "/calico-kube-controllers-metrics-tls/tls.crt"}, + corev1.EnvVar{Name: "CLIENT_COMMON_NAME", Value: monitor.PrometheusClientTLSSecretName}, + )) + Expect(c.VolumeMounts).To(ContainElement(HaveField("Name", kubecontrollers.KubeControllerPrometheusTLSSecret))) + Expect(dp.Spec.Template.Spec.Volumes).To(ContainElement(HaveField("Name", kubecontrollers.KubeControllerPrometheusTLSSecret))) + Expect(dp.Spec.Template.Annotations).NotTo(BeEmpty(), "expected the cert hash annotation") + }) + + It("adds the cert-management init container when certificate management is enabled", func() { + rc, _, err := ext.ExtendContext(certManagementControllerContext()) + Expect(err).NotTo(HaveOccurred()) + + objs, _ := applyExtensions(ext, render.ComponentNameKubeControllers, rc, []client.Object{kubeControllersDeployment()}, nil) + dp, ok := extensions.FindObject[*appsv1.Deployment](objs, kubecontrollers.KubeController) + Expect(ok).To(BeTrue()) + + Expect(dp.Spec.Template.Spec.InitContainers).To(HaveLen(1)) + Expect(dp.Spec.Template.Spec.InitContainers[0].Name).To(Equal(fmt.Sprintf("%s-key-cert-provisioner", kubecontrollers.KubeControllerPrometheusTLSSecret))) + }) +}) + +// certManagementControllerContext builds a controller context whose certificate +// manager issues cert-management (CSR-based) keypairs. +func certManagementControllerContext() extensions.ControllerContext { + scheme := runtime.NewScheme() + Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) + c := ctrlrfake.DefaultFakeClientBuilder(scheme).Build() + + ca, err := tls.MakeCA(rmeta.DefaultOperatorCASignerName()) + Expect(err).NotTo(HaveOccurred()) + caCert, _, err := ca.Config.GetPEMBytes() + Expect(err).NotTo(HaveOccurred()) + + installation := &operatorv1.InstallationSpec{ + Variant: operatorv1.CalicoEnterprise, + CertificateManagement: &operatorv1.CertificateManagement{CACert: caCert}, + } + certManager, err := certificatemanager.Create(c, installation, "", common.OperatorNamespace(), certificatemanager.AllowCACreation()) + Expect(err).NotTo(HaveOccurred()) + + return extensions.ControllerContext{ + RenderContext: extensions.RenderContext{ + Installation: installation, + FelixConfiguration: &v3.FelixConfiguration{}, + TrustedBundle: certManager.CreateTrustedBundle(), + ClusterDomain: "cluster.local", + }, + Controller: extensions.InstallationController, + Ctx: context.Background(), + Client: c, + CertificateManager: certManager, + } +} diff --git a/pkg/enterprise/register.go b/pkg/enterprise/register.go index 4d533a0e33..839e56b66a 100644 --- a/pkg/enterprise/register.go +++ b/pkg/enterprise/register.go @@ -35,6 +35,7 @@ func New() *extensions.Set { registerWindows(ent) registerGuardian(ent) registerAPIServer(ent) + registerKubeControllers(ent) // When the enterprise operator manages a Calico installation, clean up the // Enterprise objects left behind by a prior Enterprise installation. diff --git a/pkg/render/component.go b/pkg/render/component.go index 778e1a3c2f..6fdf85303a 100644 --- a/pkg/render/component.go +++ b/pkg/render/component.go @@ -76,4 +76,9 @@ const ( ComponentNameWindows = "windows" ComponentNameWindowsNodeImg = "windows-node-image" ComponentNameWindowsCNIImg = "windows-cni-image" + + // ComponentNameKubeControllers keys the calico-kube-controllers modifier. The + // es-calico-kube-controllers deployment shares the component type but leaves + // its modifier key empty, so it is not decorated. + ComponentNameKubeControllers = "kube-controllers" ) diff --git a/pkg/render/kubecontrollers/kube-controllers.go b/pkg/render/kubecontrollers/kube-controllers.go index b6340efb74..46dbf2cd8a 100644 --- a/pkg/render/kubecontrollers/kube-controllers.go +++ b/pkg/render/kubecontrollers/kube-controllers.go @@ -43,7 +43,6 @@ import ( "github.com/tigera/operator/pkg/render/common/secret" "github.com/tigera/operator/pkg/render/common/securitycontext" "github.com/tigera/operator/pkg/render/common/securitycontextconstraints" - "github.com/tigera/operator/pkg/render/monitor" "github.com/tigera/operator/pkg/tls/certificatemanagement" ) @@ -105,8 +104,6 @@ type KubeControllersConfiguration struct { WASMCACert *corev1.ConfigMap TrustedBundle certificatemanagement.TrustedBundleRO - MetricsServerTLS certificatemanagement.KeyPairInterface - // Namespace to be installed into. Namespace string @@ -172,6 +169,12 @@ type KubeControllersConfiguration struct { // webhook surface lifecycle (rendered when WAFGatewayExtensionEnabled, deleted // otherwise). Only the calico-kube-controllers component sets this. ManageWAFWebhook bool + + // ModifierKey is the extension modifier key the component reports through + // render.Extensible. calico-kube-controllers sets it so the enterprise modifier + // can layer on its metrics TLS; es-calico-kube-controllers leaves it empty so it + // is never decorated. + ModifierKey string } func NewCalicoKubeControllersPolicy(cfg *KubeControllersConfiguration, defaultDeny *v3.NetworkPolicy) render.Component { @@ -206,6 +209,7 @@ func NewCalicoKubeControllers(cfg *KubeControllersConfiguration) render.Componen cfg.RoleBindingName = KubeControllerRoleBinding cfg.MetricsName = KubeControllerMetrics cfg.ManageWAFWebhook = true + cfg.ModifierKey = render.ComponentNameKubeControllers cfg.Rules = KubeControllersRoleCommonRules(cfg) cfg.EnabledControllers = []string{"node", "loadbalancer"} @@ -374,6 +378,12 @@ func (c *kubeControllersComponent) Ready() bool { return true } +// ModifierKey implements render.Extensible. It is empty for es-calico-kube-controllers +// (never decorated) and set for calico-kube-controllers. +func (c *kubeControllersComponent) ModifierKey() string { + return c.cfg.ModifierKey +} + func KubeControllersRoleCommonRules(cfg *KubeControllersConfiguration) []rbacv1.PolicyRule { rules := []rbacv1.PolicyRule{ { @@ -739,13 +749,6 @@ func (c *kubeControllersComponent) controllersDeployment() *appsv1.Deployment { } } - if c.cfg.MetricsServerTLS != nil { - env = append(env, - corev1.EnvVar{Name: "TLS_KEY_PATH", Value: c.cfg.MetricsServerTLS.VolumeMountKeyFilePath()}, - corev1.EnvVar{Name: "TLS_CRT_PATH", Value: c.cfg.MetricsServerTLS.VolumeMountCertificateFilePath()}, - corev1.EnvVar{Name: "CLIENT_COMMON_NAME", Value: monitor.PrometheusClientTLSSecretName}, - ) - } if c.cfg.TrustedBundle != nil { env = append(env, corev1.EnvVar{Name: "CA_CRT_PATH", Value: c.cfg.TrustedBundle.MountPath()}, @@ -814,9 +817,6 @@ func (c *kubeControllersComponent) controllersDeployment() *appsv1.Deployment { } var initContainers []corev1.Container - if c.cfg.MetricsServerTLS != nil && c.cfg.MetricsServerTLS.UseCertificateManagement() { - initContainers = append(initContainers, c.cfg.MetricsServerTLS.InitContainer(c.cfg.Namespace, sc)) - } if c.cfg.WAFWebhookServerTLS != nil && c.cfg.WAFWebhookServerTLS.UseCertificateManagement() { initContainers = append(initContainers, c.cfg.WAFWebhookServerTLS.InitContainer(c.cfg.Namespace, sc)) } @@ -953,9 +953,6 @@ func (c *kubeControllersComponent) annotations() map[string]string { am = make(map[string]string) } - if c.cfg.MetricsServerTLS != nil { - am[c.cfg.MetricsServerTLS.HashAnnotationKey()] = c.cfg.MetricsServerTLS.HashAnnotationValue() - } if c.cfg.KubeControllersGatewaySecret != nil { am[render.ElasticsearchUserHashAnnotation] = rmeta.AnnotationHash(c.cfg.KubeControllersGatewaySecret.Data) } @@ -967,9 +964,6 @@ func (c *kubeControllersComponent) kubeControllersVolumeMounts() []corev1.Volume if c.cfg.TrustedBundle != nil { mounts = append(mounts, c.cfg.TrustedBundle.VolumeMounts(c.SupportedOSType())...) } - if c.cfg.MetricsServerTLS != nil { - mounts = append(mounts, c.cfg.MetricsServerTLS.VolumeMount(c.SupportedOSType())) - } if c.cfg.WAFWebhookServerTLS != nil { mounts = append(mounts, c.cfg.WAFWebhookServerTLS.VolumeMount(c.SupportedOSType())) } @@ -981,9 +975,6 @@ func (c *kubeControllersComponent) kubeControllersVolumes() []corev1.Volume { if c.cfg.TrustedBundle != nil { volumes = append(volumes, c.cfg.TrustedBundle.Volume()) } - if c.cfg.MetricsServerTLS != nil { - volumes = append(volumes, c.cfg.MetricsServerTLS.Volume()) - } if c.cfg.WAFWebhookServerTLS != nil { volumes = append(volumes, c.cfg.WAFWebhookServerTLS.Volume()) } diff --git a/pkg/render/kubecontrollers/kube-controllers_test.go b/pkg/render/kubecontrollers/kube-controllers_test.go index 8680614827..801c4b2687 100644 --- a/pkg/render/kubecontrollers/kube-controllers_test.go +++ b/pkg/render/kubecontrollers/kube-controllers_test.go @@ -49,7 +49,6 @@ import ( rtest "github.com/tigera/operator/pkg/render/common/test" "github.com/tigera/operator/pkg/render/kubecontrollers" "github.com/tigera/operator/pkg/render/testutils" - "github.com/tigera/operator/pkg/tls" "github.com/tigera/operator/pkg/tls/certificatemanagement" ) @@ -550,8 +549,6 @@ var _ = Describe("kube-controllers rendering tests", func() { Expect(dp.Spec.Template.Spec.Containers[0].Image).To(Equal("test-reg/tigera/calico:" + components.ComponentTigeraCalico.Version)) }) It("should render all calico-kube-controllers resources for a default configuration using CalicoEnterprise", func() { - var defaultMode int32 = 420 - var kubeControllerTLS certificatemanagement.KeyPairInterface expectedResources := []struct { name string ns string @@ -566,15 +563,14 @@ var _ = Describe("kube-controllers rendering tests", func() { {name: kubecontrollers.KubeControllerMetrics, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Service"}, } + // The metrics serving TLS (TLS_KEY_PATH/TLS_CRT_PATH/CLIENT_COMMON_NAME env, + // the keypair volume + mount) is layered on by the enterprise modifier, so + // the base render here carries only the trusted bundle. expectedEnv := []corev1.EnvVar{ - {Name: "TLS_KEY_PATH", Value: "/calico-kube-controllers-metrics-tls/tls.key"}, - {Name: "TLS_CRT_PATH", Value: "/calico-kube-controllers-metrics-tls/tls.crt"}, - {Name: "CLIENT_COMMON_NAME", Value: "calico-node-prometheus-client-tls"}, {Name: "CA_CRT_PATH", Value: "/etc/pki/tls/certs/tigera-ca-bundle.crt"}, } expectedVolumeMounts := []corev1.VolumeMount{ {Name: "tigera-ca-bundle", MountPath: "/etc/pki/tls/certs", ReadOnly: true}, - {Name: "calico-kube-controllers-metrics-tls", MountPath: "/calico-kube-controllers-metrics-tls", ReadOnly: true}, } expectedVolume := []corev1.Volume{ { @@ -585,34 +581,11 @@ var _ = Describe("kube-controllers rendering tests", func() { }, }, }, - { - Name: "calico-kube-controllers-metrics-tls", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: "calico-kube-controllers-metrics-tls", - DefaultMode: &defaultMode, - }, - }, - }, } - scheme := runtime.NewScheme() - Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) - cli := ctrlrfake.DefaultFakeClientBuilder(scheme).Build() - - certificateManager, err := certificatemanager.Create(cli, nil, dns.DefaultClusterDomain, common.OperatorNamespace(), certificatemanager.AllowCACreation()) - Expect(err).NotTo(HaveOccurred()) - - kubeControllerTLS, err = certificateManager.GetOrCreateKeyPair(cli, - kubecontrollers.KubeControllerPrometheusTLSSecret, - common.OperatorNamespace(), - dns.GetServiceDNSNames(kubecontrollers.KubeControllerMetrics, common.CalicoNamespace, dns.DefaultClusterDomain)) - Expect(err).NotTo(HaveOccurred()) - // Override configuration to match expected Enterprise config. instance.Variant = operatorv1.CalicoEnterprise cfg.MetricsPort = 9094 - cfg.MetricsServerTLS = kubeControllerTLS component := kubecontrollers.NewCalicoKubeControllers(&cfg) Expect(component.ResolveImages(nil)).To(BeNil()) @@ -632,10 +605,10 @@ var _ = Describe("kube-controllers rendering tests", func() { envs := dp.Spec.Template.Spec.Containers[0].Env Expect(envs).To(ContainElements(expectedEnv)) - Expect(len(dp.Spec.Template.Spec.Containers[0].VolumeMounts)).To(Equal(2)) + Expect(len(dp.Spec.Template.Spec.Containers[0].VolumeMounts)).To(Equal(1)) Expect(dp.Spec.Template.Spec.Containers[0].VolumeMounts).To(ContainElements(expectedVolumeMounts)) - Expect(len(dp.Spec.Template.Spec.Volumes)).To(Equal(2)) + Expect(len(dp.Spec.Template.Spec.Volumes)).To(Equal(1)) Expect(dp.Spec.Template.Spec.Volumes).To(ContainElements(expectedVolume)) Expect(dp.Spec.Template.Spec.Containers[0].Image).To(Equal("test-reg/tigera/calico:" + components.ComponentTigeraCalico.Version)) @@ -1302,29 +1275,6 @@ var _ = Describe("kube-controllers rendering tests", func() { ) }) - It("should render init containers when certificate management is enabled", func() { - instance.Variant = operatorv1.CalicoEnterprise - cfg.MetricsPort = 9094 - ca, _ := tls.MakeCA(rmeta.DefaultOperatorCASignerName()) - cert, _, _ := ca.Config.GetPEMBytes() // create a valid pem block - cfg.Installation.CertificateManagement = &operatorv1.CertificateManagement{CACert: cert} - - certificateManager, err := certificatemanager.Create(cli, cfg.Installation, dns.DefaultClusterDomain, common.OperatorNamespace(), certificatemanager.AllowCACreation()) - Expect(err).NotTo(HaveOccurred()) - - tls, err := certificateManager.GetOrCreateKeyPair(cli, kubecontrollers.KubeControllerPrometheusTLSSecret, common.OperatorNamespace(), []string{""}) - Expect(err).NotTo(HaveOccurred()) - - cfg.MetricsServerTLS = tls - - resources, _ := kubecontrollers.NewCalicoKubeControllers(&cfg).Objects() - - dp := rtest.GetResource(resources, kubecontrollers.KubeController, common.CalicoNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) - Expect(dp.Spec.Template.Spec.InitContainers).To(HaveLen(1)) - csrInitContainer := dp.Spec.Template.Spec.InitContainers[0] - Expect(csrInitContainer.Name).To(Equal(fmt.Sprintf("%v-key-cert-provisioner", kubecontrollers.KubeControllerPrometheusTLSSecret))) - }) - It("should add egress policy with Enterprise variant and K8SServiceEndpoint defined", func() { cfg.K8sServiceEp.Host = "k8shost" cfg.K8sServiceEp.Port = "1234" From 8d2ff05ea6e6af8f29c2f98bd6f48d6bbf78ee8d Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Thu, 18 Jun 2026 16:28:45 -0700 Subject: [PATCH 38/38] Move the calico-kube-controllers enterprise surface into the extension calico-kube-controllers renders as pure OSS now: the common rules plus the node and loadbalancer controllers, no IsEnterprise. The enterprise extension layers on the rest through a modifier - the enterprise RBAC, the service/federatedservices/usage controllers, the metrics serving TLS, and the WAF v3 (Gateway API add-on) surface (the WASM env, the in-process admission webhook, and the network policy ingress rule). The installation hook produces the controller-side inputs the modifier can't (the webhook keypair, the merged wasm pull secret, the resolved wasm image, the operator CA) and hands them over through the render context; the WASM image resolves with the same GetReference the base uses, via the ImageSet the hook reads itself. The WAF/WASM symbols and the es-kube-controllers pull-secret helper move to pkg/enterprise. The base kube-controllers config no longer carries any WAF/WASM fields. --- .../installation/core_controller.go | 86 +-- pkg/enterprise/installation.go | 35 +- pkg/enterprise/kubecontrollers.go | 534 +++++++++++++++++- pkg/enterprise/kubecontrollers_test.go | 168 ++++++ .../waf_pull_secret_test.go | 20 +- pkg/render/component.go | 4 + .../kubecontrollers/kube-controllers.go | 375 +----------- .../kubecontrollers/kube-controllers_test.go | 275 +-------- pkg/render/kubecontrollers/waf_pull_secret.go | 101 ---- .../logstorage/esgateway/esgateway_test.go | 20 +- 10 files changed, 756 insertions(+), 862 deletions(-) rename pkg/{render/kubecontrollers => enterprise}/waf_pull_secret_test.go (88%) delete mode 100644 pkg/render/kubecontrollers/waf_pull_secret.go diff --git a/pkg/controller/installation/core_controller.go b/pkg/controller/installation/core_controller.go index 6c5686fee8..b01f01cfca 100644 --- a/pkg/controller/installation/core_controller.go +++ b/pkg/controller/installation/core_controller.go @@ -63,7 +63,6 @@ import ( "github.com/tigera/operator/pkg/common/discovery" "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/controller/certificatemanager" - "github.com/tigera/operator/pkg/controller/gatewayapi" "github.com/tigera/operator/pkg/controller/ippool" "github.com/tigera/operator/pkg/controller/k8sapi" "github.com/tigera/operator/pkg/controller/migration" @@ -74,12 +73,10 @@ import ( "github.com/tigera/operator/pkg/controller/utils" "github.com/tigera/operator/pkg/controller/utils/imageset" "github.com/tigera/operator/pkg/ctrlruntime" - "github.com/tigera/operator/pkg/dns" "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/imports/admission" "github.com/tigera/operator/pkg/imports/crds" "github.com/tigera/operator/pkg/render" - "github.com/tigera/operator/pkg/render/applicationlayer" rcertificatemanagement "github.com/tigera/operator/pkg/render/certificatemanagement" "github.com/tigera/operator/pkg/render/common/networkpolicy" "github.com/tigera/operator/pkg/render/common/resourcequota" @@ -207,13 +204,6 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { } // Watch for changes to KubeControllersConfiguration. - // Watch GatewayAPI: spec.extensions.waf.state gates the WAF v3 surface on - // calico-kube-controllers. See design tigera/designs#25 (PMREQ-384) §Gating. - if err := c.WatchObject(&operatorv1.GatewayAPI{}, &handler.EnqueueRequestForObject{}); err != nil { - log.V(5).Info("Failed to create GatewayAPI watch", "err", err) - return fmt.Errorf("core-controller failed to watch operator GatewayAPI resource: %w", err) - } - err = c.WatchObject(&v3.KubeControllersConfiguration{}, &handler.EnqueueRequestForObject{}) if err != nil { return fmt.Errorf("tigera-installation-controller failed to watch KubeControllersConfiguration resource: %w", err) @@ -1261,47 +1251,10 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile } - // Read the GatewayAPI CR (if present) to decide whether to render the WAF - // v3 (Gateway API add-on) surface — env vars, RBAC, applicationlayer - // reconciler, and the in-process admission webhook — on - // calico-kube-controllers. Default-off: if no GatewayAPI CR exists or - // spec.extensions.waf.state != Enabled, the WAF surface is not rendered. - // See design tigera/designs#25 (PMREQ-384) §Gating. - wafGatewayExtensionEnabled := false - if gatewayAPI, msg, err := gatewayapi.GetGatewayAPI(ctx, r.client); err == nil { - wafGatewayExtensionEnabled = gatewayAPI.Spec.IsWAFGatewayExtensionEnabled() - } else if !apierrors.IsNotFound(err) { - // Mirrors the GatewayAPI controller's handling: a read error or a - // duplicate default/tigera-secure pair degrades rather than guessing. - r.status.SetDegraded(operatorv1.ResourceReadError, msg, err, reqLogger) - return reconcile.Result{}, err - } - - // When the WAF v3 surface is enabled, issue the serving cert for the - // in-process WAF admission webhook (hosted by calico-kube-controllers, - // fronted by the tigera-waf-webhook Service). It is materialized into - // calico-system alongside the other kube-controllers certs below and mounted - // into the Pod by the kube-controllers render. - var wafWebhookTLS certificatemanagement.KeyPairInterface - if wafGatewayExtensionEnabled { - wafWebhookTLS, err = certificateManager.GetOrCreateKeyPair( - r.client, - applicationlayer.WAFWebhookServerTLSSecretName, - common.OperatorNamespace(), - dns.GetServiceDNSNames(applicationlayer.WAFWebhookServiceName, common.CalicoNamespace, r.opts.ClusterDomain)) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceCreateError, "Error creating WAF admission webhook TLS certificate", err, reqLogger) - return reconcile.Result{}, err - } - } - keyPairOptions := []rcertificatemanagement.KeyPairOption{ rcertificatemanagement.NewKeyPairOption(typhaNodeTLS.NodeSecret, true, true), rcertificatemanagement.NewKeyPairOption(typhaNodeTLS.TyphaSecret, true, true), rcertificatemanagement.NewKeyPairOption(typhaNodeTLS.TyphaSecretNonClusterHost, true, true), - // Nil when the WAF v3 surface is disabled; the certificate-management - // render skips nil key pairs. - rcertificatemanagement.NewKeyPairOption(wafWebhookTLS, true, true), } // Manage any key pairs the variant extension created controller-side. for _, kp := range managedKeyPairs { @@ -1542,33 +1495,9 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile } components = append(components, render.CSI(&csiCfg)) - // Build a configuration for rendering calico/kube-controllers. - // Provision a dedicated WAF wasm pull secret so the WAF reconciler - // replicates it into tenant namespaces without clashing with the - // operator-managed tigera-pull-secret the GatewayAPI render also copies - // there (EV-6386). The EnvoyExtensionPolicy image source takes a single - // pullSecretRef, so the registry auths of all Installation pull secrets - // are merged into it rather than picking one. - var wasmPullSecret *corev1.Secret - if wafGatewayExtensionEnabled && len(pullSecrets) > 0 { - var skipped []string - wasmPullSecret, skipped = kubecontrollers.MergeWAFPullSecret(pullSecrets) - if len(skipped) > 0 { - reqLogger.Info("Skipped unparseable imagePullSecrets when building the WAF wasm pull secret", "skipped", skipped) - } - } - // Provision the dedicated WAF wasm CA-bundle ConfigMap as a renamed copy of - // the trusted CA bundle, so the WAF reconciler replicates it into tenant - // namespaces for the Coraza wasm OCI registry TLS check without clashing with - // the operator-managed tigera-ca-bundle the GatewayAPI render also copies - // there (EV-6386). The dedicated source was previously a TODO; the full - // TrustedBundle (not the RO interface the kube-controllers render sees) is - // available here, so build it in the core controller. - var wasmCACert *corev1.ConfigMap - if wafGatewayExtensionEnabled { - wasmCACert = typhaNodeTLS.TrustedBundle.ConfigMap(common.CalicoNamespace) - wasmCACert.Name = kubecontrollers.WASMCACertName - } + // Build a configuration for rendering calico/kube-controllers. The Calico + // Enterprise surface (extra RBAC, enterprise controllers, metrics TLS, and the + // WAF v3 / Gateway API add-on) is layered on by the enterprise extension. kubeControllersCfg := kubecontrollers.KubeControllersConfiguration{ K8sServiceEp: k8sapi.Endpoint, K8sServiceEpPodNetwork: k8sapi.PodNetworkEndpoint, @@ -1581,15 +1510,6 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile TrustedBundle: typhaNodeTLS.TrustedBundle, Namespace: common.CalicoNamespace, BindingNamespaces: []string{common.CalicoNamespace}, - WAFGatewayExtensionEnabled: wafGatewayExtensionEnabled, - WAFWebhookServerTLS: wafWebhookTLS, - WASMPullSecret: wasmPullSecret, - WASMCACert: wasmCACert, - // The webhook Service + ValidatingWebhookConfiguration are rendered by - // the kube-controllers component (and deleted when the WAF extension is - // disabled); the caBundle is the operator CA that issued the serving - // cert above. - WAFWebhookCABundle: certificateManager.KeyPair().GetCertificatePEM(), } components = append(components, kubecontrollers.NewCalicoKubeControllers(&kubeControllersCfg)) diff --git a/pkg/enterprise/installation.go b/pkg/enterprise/installation.go index 12eef67b13..7df2d2bf30 100644 --- a/pkg/enterprise/installation.go +++ b/pkg/enterprise/installation.go @@ -17,6 +17,7 @@ package enterprise import ( "fmt" + rbacv1 "k8s.io/api/rbac/v1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/handler" @@ -50,6 +51,13 @@ type installationRenderData struct { // collectProcessPath mirrors LogCollector.Spec.CollectProcessPath being // enabled; the node modifier uses it to set HostPID and the felix env. collectProcessPath bool + + // calico-kube-controllers enterprise additions the kube-controllers modifier + // applies: the enterprise cluster role rules, the enterprise enabled controllers, + // and the WAF v3 (Gateway API add-on) surface. + kubeControllerRules []rbacv1.PolicyRule + kubeControllerControllers []string + waf wafRenderData } // installationData pulls the installation extension's render data back out of the @@ -77,6 +85,8 @@ func (coreControllerExtension) Watches(c ctrlruntime.Controller) error { &operatorv1.ManagementCluster{}, &operatorv1.ManagementClusterConnection{}, &operatorv1.LogCollector{}, + // GatewayAPI.spec.extensions.waf.state gates the WAF v3 surface on calico-kube-controllers. + &operatorv1.GatewayAPI{}, } { if err := c.WatchObject(obj, &handler.EnqueueRequestForObject{}); err != nil { return err @@ -126,10 +136,26 @@ func (coreControllerExtension) ExtendContext(cc extensions.ControllerContext) (e if err != nil { return rc, nil, fmt.Errorf("error reading LogCollector: %w", err) } + + // calico-kube-controllers enterprise additions: the WAF surface, the enterprise + // cluster role rules, and the enterprise enabled controllers. A managed cluster's + // kube-controllers needs an extra license-push rule. + managementClusterConnection, err := utils.GetManagementClusterConnection(cc.Ctx, cc.Client) + if err != nil { + return rc, nil, fmt.Errorf("error reading ManagementClusterConnection: %w", err) + } + waf, wafWebhookTLS, err := buildWAFData(cc) + if err != nil { + return rc, nil, fmt.Errorf("error preparing WAF configuration: %w", err) + } + rc.Extension = installationRenderData{ - nodePrometheusTLS: nodePrometheusTLS, - kubeControllerTLS: kubeControllerTLS, - collectProcessPath: collectProcessPathEnabled(logCollector), + nodePrometheusTLS: nodePrometheusTLS, + kubeControllerTLS: kubeControllerTLS, + collectProcessPath: collectProcessPathEnabled(logCollector), + kubeControllerRules: calicoKubeControllersEnterpriseRules(waf.enabled, managementClusterConnection != nil), + kubeControllerControllers: calicoKubeControllersEnterpriseControllers(waf.enabled), + waf: waf, } prometheusClientCert, err := cc.CertificateManager.GetCertificate(cc.Client, monitor.PrometheusClientTLSSecretName, common.OperatorNamespace()) @@ -165,5 +191,8 @@ func (coreControllerExtension) ExtendContext(cc extensions.ControllerContext) (e if kubeControllerTLS != nil { managed = append(managed, kubeControllerTLS) } + if wafWebhookTLS != nil { + managed = append(managed, wafWebhookTLS) + } return rc, managed, nil } diff --git a/pkg/enterprise/kubecontrollers.go b/pkg/enterprise/kubecontrollers.go index 884f928750..2213e4c1d6 100644 --- a/pkg/enterprise/kubecontrollers.go +++ b/pkg/enterprise/kubecontrollers.go @@ -15,73 +15,202 @@ package enterprise import ( + "encoding/json" + "fmt" + "path/filepath" + "strings" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/controller/gatewayapi" + "github.com/tigera/operator/pkg/controller/utils" + "github.com/tigera/operator/pkg/controller/utils/imageset" + "github.com/tigera/operator/pkg/dns" "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/render/applicationlayer" relasticsearch "github.com/tigera/operator/pkg/render/common/elasticsearch" rmeta "github.com/tigera/operator/pkg/render/common/meta" "github.com/tigera/operator/pkg/render/common/networkpolicy" + "github.com/tigera/operator/pkg/render/common/secret" "github.com/tigera/operator/pkg/render/kubecontrollers" "github.com/tigera/operator/pkg/render/monitor" + "github.com/tigera/operator/pkg/tls/certificatemanagement" "github.com/tigera/operator/pkg/url" ) -// registerKubeControllers registers the calico-kube-controllers modifier. There is +// registerKubeControllers registers the calico-kube-controllers modifiers. There is // no image override: kube-controllers runs from the combined calico image, which // resolves by variant in the base render. func registerKubeControllers(v *extensions.Variant) { v.Modify(render.ComponentNameKubeControllers, modifyKubeControllers) + v.Modify(render.ComponentNameKubeControllersPolicy, modifyKubeControllersPolicy) } -// modifyKubeControllers layers the Calico Enterprise metrics serving TLS onto the -// rendered calico-kube-controllers deployment: the env pointing at the keypair, the -// volume + mount, the cert-management init container (when in use), and the pod hash -// annotation that rolls the pod on cert rotation. The keypair has cluster side -// effects, so the installation extension creates it and hands it in via rc. In core -// (calico) it is never created, so the base deployment carries no metrics TLS. -func modifyKubeControllers(rc extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { - tls := installationData(rc).kubeControllerTLS - if tls == nil { +// modifyKubeControllersPolicy adds the WAF admission webhook ingress rule to the +// calico-kube-controllers calico-system network policy, so the kube-apiserver can +// reach the in-process webhook on :9443 (EV-6386). Without it the calico-system +// default-deny drops the apiserver->:9443 call and WAF admission times out. +func modifyKubeControllersPolicy(rc extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { + if !installationData(rc).waf.enabled { return objs, del } - - dp, ok := extensions.FindObject[*appsv1.Deployment](objs, kubecontrollers.KubeController) + policy, ok := extensions.FindObject[*v3.NetworkPolicy](objs, kubecontrollers.KubeControllerNetworkPolicyName) if !ok { return objs, del } + policy.Spec.Ingress = append(policy.Spec.Ingress, v3.Rule{ + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: v3.EntityRule{ + Ports: networkpolicy.Ports(uint16(applicationlayer.WAFWebhookContainerPort)), + }, + }) + return objs, del +} + +// modifyKubeControllers layers the full Calico Enterprise surface onto the rendered +// calico-kube-controllers objects: the enterprise cluster role rules, the enterprise +// enabled controllers, the metrics serving TLS, and the WAF v3 (Gateway API add-on) +// surface. The modifier only runs for the enterprise variant, so everything it adds +// is enterprise-only by construction - the base render carries none of it. The +// controller-side inputs (keypairs, the resolved wasm image, the pull secret) are +// produced by the installation hook and handed in through rc. +func modifyKubeControllers(rc extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { + data := installationData(rc) + + if role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, kubecontrollers.KubeControllerRole); ok { + role.Rules = append(role.Rules, data.kubeControllerRules...) + } + + if dp, ok := extensions.FindObject[*appsv1.Deployment](objs, kubecontrollers.KubeController); ok { + modifyKubeControllersDeployment(rc, dp, data) + } + + // The WAF admission webhook surface (Service + ValidatingWebhookConfiguration), + // the wasm pull secret, and the wasm CA bundle. Created when WAF is enabled, + // deleted otherwise so toggling the extension off cleans them up. + webhookObjs := applicationlayer.WAFAdmissionWebhookComponents(data.waf.caBundle) + if data.waf.enabled { + objs = append(objs, webhookObjs...) + if data.waf.pullSecret != nil { + objs = append(objs, secret.ToRuntimeObjects(secret.CopyToNamespace(common.CalicoNamespace, data.waf.pullSecret)...)...) + } + if data.waf.caCert != nil { + objs = append(objs, data.waf.caCert) + } + } else { + del = append(del, webhookObjs...) + } + + return objs, del +} + +func modifyKubeControllersDeployment(rc extensions.RenderContext, dp *appsv1.Deployment, data installationRenderData) { spec := &dp.Spec.Template.Spec - spec.Volumes = append(spec.Volumes, tls.Volume()) + if dp.Spec.Template.Annotations == nil { + dp.Spec.Template.Annotations = map[string]string{} + } + + if tls := data.kubeControllerTLS; tls != nil { + spec.Volumes = append(spec.Volumes, tls.Volume()) + dp.Spec.Template.Annotations[tls.HashAnnotationKey()] = tls.HashAnnotationValue() + } + if waf := data.waf; waf.enabled && waf.webhookTLS != nil { + spec.Volumes = append(spec.Volumes, waf.webhookTLS.Volume()) + } for i := range spec.Containers { c := &spec.Containers[i] if c.Name != kubecontrollers.KubeController { continue } - c.Env = append(c.Env, - corev1.EnvVar{Name: "TLS_KEY_PATH", Value: tls.VolumeMountKeyFilePath()}, - corev1.EnvVar{Name: "TLS_CRT_PATH", Value: tls.VolumeMountCertificateFilePath()}, - corev1.EnvVar{Name: "CLIENT_COMMON_NAME", Value: monitor.PrometheusClientTLSSecretName}, - ) - c.VolumeMounts = append(c.VolumeMounts, tls.VolumeMount(rmeta.OSTypeLinux)) - if tls.UseCertificateManagement() { - spec.InitContainers = append(spec.InitContainers, tls.InitContainer(common.CalicoNamespace, c.SecurityContext)) + + appendEnabledControllers(c, data.kubeControllerControllers) + c.Env = append(c.Env, enterpriseEnv(rc)...) + + if tls := data.kubeControllerTLS; tls != nil { + c.Env = append(c.Env, + corev1.EnvVar{Name: "TLS_KEY_PATH", Value: tls.VolumeMountKeyFilePath()}, + corev1.EnvVar{Name: "TLS_CRT_PATH", Value: tls.VolumeMountCertificateFilePath()}, + corev1.EnvVar{Name: "CLIENT_COMMON_NAME", Value: monitor.PrometheusClientTLSSecretName}, + ) + c.VolumeMounts = append(c.VolumeMounts, tls.VolumeMount(rmeta.OSTypeLinux)) + if tls.UseCertificateManagement() { + spec.InitContainers = append(spec.InitContainers, tls.InitContainer(common.CalicoNamespace, c.SecurityContext)) + } + } + + if waf := data.waf; waf.enabled { + c.Env = append(c.Env, wafEnv(waf)...) + c.Ports = append(c.Ports, corev1.ContainerPort{ + Name: "waf-webhook", + ContainerPort: applicationlayer.WAFWebhookContainerPort, + Protocol: corev1.ProtocolTCP, + }) + if waf.webhookTLS != nil { + c.VolumeMounts = append(c.VolumeMounts, waf.webhookTLS.VolumeMount(rmeta.OSTypeLinux)) + if waf.webhookTLS.UseCertificateManagement() { + spec.InitContainers = append(spec.InitContainers, waf.webhookTLS.InitContainer(common.CalicoNamespace, c.SecurityContext)) + } + } } } +} - if dp.Spec.Template.Annotations == nil { - dp.Spec.Template.Annotations = map[string]string{} +// appendEnabledControllers folds the enterprise controllers into the existing +// ENABLED_CONTROLLERS env the base render set (node,loadbalancer). +func appendEnabledControllers(c *corev1.Container, extra []string) { + if len(extra) == 0 { + return + } + for i := range c.Env { + if c.Env[i].Name == "ENABLED_CONTROLLERS" { + c.Env[i].Value = c.Env[i].Value + "," + strings.Join(extra, ",") + return + } } - dp.Spec.Template.Annotations[tls.HashAnnotationKey()] = tls.HashAnnotationValue() +} - return objs, del +// enterpriseEnv is the static enterprise env for calico-kube-controllers. The +// modifier runs only for the enterprise variant, so these are never rendered for core. +func enterpriseEnv(rc extensions.RenderContext) []corev1.EnvVar { + var env []corev1.EnvVar + if rc.TrustedBundle != nil { + env = append(env, corev1.EnvVar{Name: "MULTI_CLUSTER_FORWARDING_CA", Value: rc.TrustedBundle.MountPath()}) + } + if in := rc.Installation; in != nil && in.CalicoNetwork != nil && in.CalicoNetwork.MultiInterfaceMode != nil { + env = append(env, corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: in.CalicoNetwork.MultiInterfaceMode.Value()}) + } + return env +} + +// wafEnv is the WAF v3 env the kube-controllers binary consumes to program WAF policy +// attachments. WASM_IMAGE is the pre-resolved reference the hook produced. +func wafEnv(waf wafRenderData) []corev1.EnvVar { + var env []corev1.EnvVar + if waf.wasmImage != "" { + env = append(env, corev1.EnvVar{Name: "WASM_IMAGE", Value: waf.wasmImage}) + } + if waf.pullSecret != nil { + env = append(env, corev1.EnvVar{Name: "WASM_PULL_SECRET", Value: waf.pullSecret.Name}) + } + if waf.caCert != nil { + env = append(env, corev1.EnvVar{Name: "WASM_CA_CERT", Value: waf.caCert.Name}) + } + if waf.webhookTLS != nil { + env = append(env, corev1.EnvVar{Name: "WAF_WEBHOOK_CERT_DIR", Value: filepath.Dir(waf.webhookTLS.VolumeMountCertificateFilePath())}) + } + return env } const ( @@ -95,6 +224,18 @@ const ( ElasticsearchKubeControllersUserName = "tigera-ee-kube-controllers" ElasticsearchKubeControllersSecureUserSecret = "tigera-ee-kube-controllers-elasticsearch-access-gateway" ElasticsearchKubeControllersVerificationUserSecret = "tigera-ee-kube-controllers-gateway-verification-credentials" + + // WASMPullSecretName is the dedicated image-pull Secret (a merged copy of the + // install pull secrets) the WAF reconciler replicates into tenant namespaces for + // the Coraza wasm OCI pull. A dedicated name avoids clashing with the + // operator-managed tigera-pull-secret the GatewayAPI render also copies there (EV-6386). + WASMPullSecretName = "tigera-waf-pull-secret" + + // WASMCACertName is the dedicated CA-bundle ConfigMap the WAF reconciler + // replicates into tenant namespaces for the Coraza wasm OCI registry TLS check - + // a dedicated name avoids clashing with the operator-managed tigera-ca-bundle the + // GatewayAPI render also copies there (EV-6386). It is a renamed copy of the trusted bundle. + WASMCACertName = "tigera-waf-ca-bundle" ) // NewElasticsearchKubeControllers fills the generic kube-controllers configuration @@ -111,7 +252,7 @@ func NewElasticsearchKubeControllers(cfg *kubecontrollers.KubeControllersConfigu cfg.DisableConfigAPI = cfg.Tenant.MultiTenant() cfg.Rules = kubecontrollers.KubeControllersRoleCommonRules(cfg) - cfg.Rules = append(cfg.Rules, kubecontrollers.KubeControllersRoleEnterpriseCommonRules(cfg)...) + cfg.Rules = append(cfg.Rules, kubeControllersEnterpriseCommonRules(false, cfg.ManagementClusterConnection != nil)...) cfg.Rules = append(cfg.Rules, rbacv1.PolicyRule{ APIGroups: []string{"elasticsearch.k8s.elastic.co"}, @@ -181,6 +322,200 @@ func esKubeControllersEnv(cfg *kubecontrollers.KubeControllersConfiguration) []c return env } +// kubeControllersEnterpriseCommonRules are the Calico Enterprise cluster role rules +// shared by calico-kube-controllers and es-calico-kube-controllers. wafEnabled adds +// the WAF v3 (Gateway API add-on) rules; managedCluster adds the license-push rule a +// managed cluster's kube-controllers needs. +func kubeControllersEnterpriseCommonRules(wafEnabled, managedCluster bool) []rbacv1.PolicyRule { + rules := []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{"configmaps"}, + Verbs: []string{"watch", "list", "get", "update", "create", "delete"}, + }, + { + // The Federated Services Controller needs access to the remote kubeconfig secret + // in order to create a remote syncer. + APIGroups: []string{""}, + Resources: []string{"secrets"}, + Verbs: []string{"watch", "list", "get"}, + }, + { + // Needed to validate the license + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"licensekeys"}, + Verbs: []string{"get", "watch", "list"}, + }, + { + // Needed to update the status of the LicenseKey with the result of license validation. + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"licensekeys/status"}, + Verbs: []string{"update"}, + }, + { + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"deeppacketinspections"}, + Verbs: []string{"get", "watch", "list"}, + }, + { + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"deeppacketinspections/status"}, + Verbs: []string{"update"}, + }, + { + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"packetcaptures"}, + Verbs: []string{"get", "list", "update"}, + }, + { + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"packetcaptures/status"}, + Verbs: []string{"update"}, + }, + } + + if wafEnabled { + rules = append(rules, wafRules()...) + } + + if managedCluster { + rules = append(rules, + rbacv1.PolicyRule{ + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"licensekeys"}, + Verbs: []string{"get", "create", "update", "list", "watch"}, + }, + ) + } + + return rules +} + +// calicoKubeControllersEnterpriseRules are the enterprise cluster role rules layered +// onto calico-kube-controllers: the shared enterprise rules plus the calico-specific +// ones (federated endpoints, license usage reporting). +func calicoKubeControllersEnterpriseRules(wafEnabled, managedCluster bool) []rbacv1.PolicyRule { + rules := kubeControllersEnterpriseCommonRules(wafEnabled, managedCluster) + return append(rules, + rbacv1.PolicyRule{ + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"remoteclusterconfigurations"}, + Verbs: []string{"watch", "list", "get"}, + }, + rbacv1.PolicyRule{ + APIGroups: []string{""}, + Resources: []string{"endpoints"}, + Verbs: []string{"create", "update", "delete"}, + }, + rbacv1.PolicyRule{ + APIGroups: []string{""}, + Resources: []string{"namespaces"}, + Verbs: []string{"get"}, + }, + rbacv1.PolicyRule{ + APIGroups: []string{"usage.tigera.io"}, + Resources: []string{"licenseusagereports"}, + Verbs: []string{"create", "update", "delete", "watch", "list", "get"}, + }, + ) +} + +// calicoKubeControllersEnterpriseControllers are the enterprise controllers added to +// the calico-kube-controllers ENABLED_CONTROLLERS list (on top of the base +// node,loadbalancer). applicationlayer is added only when the WAF extension is on. +func calicoKubeControllersEnterpriseControllers(wafEnabled bool) []string { + controllers := []string{"service", "federatedservices", "usage"} + if wafEnabled { + controllers = append(controllers, "applicationlayer") + } + return controllers +} + +// wafRules are the WAF v3 (Gateway API add-on) cluster role rules, gated by +// GatewayAPI.spec.extensions.waf.state == Enabled. +func wafRules() []rbacv1.PolicyRule { + return []rbacv1.PolicyRule{ + // Application-layer (gateway-addons) reconcilers reconcile WAF resources + // against Gateway API targetRefs and emit events on the policy objects. + { + APIGroups: []string{"applicationlayer.projectcalico.org"}, + Resources: []string{ + "wafpolicies", "globalwafpolicies", + "wafplugins", "globalwafplugins", + "wafvalidationpolicies", "globalwafvalidationpolicies", + }, + Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, + }, + { + APIGroups: []string{"applicationlayer.projectcalico.org"}, + Resources: []string{ + "wafpolicies/status", "globalwafpolicies/status", + "wafplugins/status", "globalwafplugins/status", + "wafvalidationpolicies/status", "globalwafvalidationpolicies/status", + }, + Verbs: []string{"get", "update", "patch"}, + }, + { + APIGroups: []string{"applicationlayer.projectcalico.org"}, + Resources: []string{ + "wafpolicies/finalizers", "globalwafpolicies/finalizers", + "wafplugins/finalizers", "globalwafplugins/finalizers", + "wafvalidationpolicies/finalizers", "globalwafvalidationpolicies/finalizers", + }, + Verbs: []string{"update"}, + }, + { + // Validate Gateway API targetRefs and surface attachment status. + APIGroups: []string{"gateway.networking.k8s.io"}, + Resources: []string{"gateways", "httproutes", "tcproutes", "tlsroutes", "grpcroutes"}, + Verbs: []string{"get", "list", "watch", "update", "patch"}, + }, + { + APIGroups: []string{"gateway.networking.k8s.io"}, + Resources: []string{"gateways/status", "httproutes/status", "tcproutes/status", "tlsroutes/status", "grpcroutes/status"}, + Verbs: []string{"get", "update", "patch"}, + }, + // controller-runtime Reconcilers (e.g. the applicationlayer manager) record + // events on watched objects via Recorder.Eventf; both core and events.k8s.io + // API groups are emitted depending on the kubernetes version. + { + APIGroups: []string{""}, + Resources: []string{"events"}, + Verbs: []string{"create", "patch"}, + }, + { + APIGroups: []string{"events.k8s.io"}, + Resources: []string{"events"}, + Verbs: []string{"create", "patch"}, + }, + // Application-layer reconciler replicates the WAF wasm pull Secret from + // the controller namespace (calico-system) into each WAFPolicy's + // namespace so the rendered EnvoyExtensionPolicy can reference it. Also + // replicates CA-cert ConfigMaps when WASM_CA_CERT is set. + { + APIGroups: []string{""}, + Resources: []string{"secrets", "configmaps"}, + Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, + }, + // Application-layer reconciler emits one EnvoyExtensionPolicy per WAF + // targetRef to bind the Coraza wasm filter at the gateway / route. + { + APIGroups: []string{"gateway.envoyproxy.io"}, + Resources: []string{"envoyextensionpolicies"}, + Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, + }, + // Application-layer reconciler stamps each namespace with its allocated WAF + // rule-id range (applicationlayer.projectcalico.org/waf-id-range annotation) + // so application operators can author in-range rules. The base role already + // grants namespaces get/list/watch; the annotation write needs patch/update. + { + APIGroups: []string{""}, + Resources: []string{"namespaces"}, + Verbs: []string{"get", "patch", "update"}, + }, + } +} + func esKubeControllersCalicoSystemPolicy(cfg *kubecontrollers.KubeControllersConfiguration) *v3.NetworkPolicy { if cfg.ManagementClusterConnection != nil { return nil @@ -230,3 +565,150 @@ func esKubeControllersCalicoSystemPolicy(cfg *kubecontrollers.KubeControllersCon }, } } + +// wafRenderData is the controller-produced WAF v3 (Gateway API add-on) state the +// installation hook hands the kube-controllers modifier through the render context. +// The zero value (enabled false) means the modifier deletes the webhook objects. +type wafRenderData struct { + enabled bool + wasmImage string + pullSecret *corev1.Secret + caCert *corev1.ConfigMap + webhookTLS certificatemanagement.KeyPairInterface + caBundle []byte +} + +// buildWAFData reads the GatewayAPI CR and, when the WAF extension is enabled, +// produces everything the modifier needs that it can't compute itself: the resolved +// wasm image, the webhook serving keypair (also returned as a managed keypair), the +// merged wasm pull secret, the wasm CA bundle ConfigMap, and the operator CA PEM. +func buildWAFData(cc extensions.ControllerContext) (wafRenderData, certificatemanagement.KeyPairInterface, error) { + gw, _, err := gatewayapi.GetGatewayAPI(cc.Ctx, cc.Client) + if err != nil && !apierrors.IsNotFound(err) { + return wafRenderData{}, nil, err + } + if gw == nil || !gw.Spec.IsWAFGatewayExtensionEnabled() { + return wafRenderData{}, nil, nil + } + + in := cc.Installation + // The wasm is baked into the gateway envoy-proxy image. Resolve it with the same + // GetReference the base render uses for every image; the hook has the ImageSet here. + imageSet, err := imageset.GetImageSet(cc.Ctx, cc.Client, in.Variant) + if err != nil { + return wafRenderData{}, nil, err + } + wasmImage, err := components.GetReference(components.ComponentGatewayAPIEnvoyProxy, in.Registry, in.ImagePath, in.ImagePrefix, imageSet) + if err != nil { + return wafRenderData{}, nil, err + } + + webhookTLS, err := cc.CertificateManager.GetOrCreateKeyPair( + cc.Client, + applicationlayer.WAFWebhookServerTLSSecretName, + common.OperatorNamespace(), + dns.GetServiceDNSNames(applicationlayer.WAFWebhookServiceName, common.CalicoNamespace, cc.ClusterDomain), + ) + if err != nil { + return wafRenderData{}, nil, err + } + + pullSecrets, err := utils.GetInstallationPullSecrets(in, cc.Client) + if err != nil { + return wafRenderData{}, nil, err + } + var pullSecret *corev1.Secret + if len(pullSecrets) > 0 { + pullSecret, _ = MergeWAFPullSecret(pullSecrets) + } + + var caCert *corev1.ConfigMap + if cc.TrustedBundle != nil { + caCert = cc.TrustedBundle.ConfigMap(common.CalicoNamespace) + caCert.Name = WASMCACertName + } + + return wafRenderData{ + enabled: true, + wasmImage: wasmImage, + pullSecret: pullSecret, + caCert: caCert, + webhookTLS: webhookTLS, + caBundle: cc.CertificateManager.KeyPair().GetCertificatePEM(), + }, webhookTLS, nil +} + +// MergeWAFPullSecret synthesizes the dedicated WAF wasm pull secret +// (tigera-waf-pull-secret) by merging the registry auths of every Installation pull +// secret. The EnvoyExtensionPolicy image source takes a single pullSecretRef, so a +// merged secret is the only way to honor multiple Installation pull secrets for the +// Coraza wasm OCI pull (e.g. the Tigera pull secret plus a private registry mirror). +// +// If the same registry appears in more than one secret, the first secret in +// Installation order wins. Secrets that cannot be parsed are skipped and their names +// returned, so the caller can log them without failing the reconcile. Returns a nil +// Secret when no registry auths could be collected. +func MergeWAFPullSecret(pullSecrets []*corev1.Secret) (*corev1.Secret, []string) { + merged := map[string]json.RawMessage{} + var skipped []string + for _, s := range pullSecrets { + auths, err := registryAuths(s) + if err != nil { + skipped = append(skipped, s.Name) + continue + } + for registry, auth := range auths { + if _, ok := merged[registry]; !ok { + merged[registry] = auth + } + } + } + if len(merged) == 0 { + return nil, skipped + } + + // Marshalling a map sorts its keys, so the rendered bytes are deterministic and + // do not churn the object on every reconcile. + data, err := json.Marshal(map[string]map[string]json.RawMessage{"auths": merged}) + if err != nil { + // Each auth entry round-trips from a successful Unmarshal above, so this + // cannot fail in practice; treat it as nothing to render. + return nil, skipped + } + + return &corev1.Secret{ + TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{Name: WASMPullSecretName, Namespace: common.CalicoNamespace}, + Type: corev1.SecretTypeDockerConfigJson, + Data: map[string][]byte{corev1.DockerConfigJsonKey: data}, + }, skipped +} + +// registryAuths extracts the per-registry auth entries from a pull secret of either +// the dockerconfigjson type (auths nested under an "auths" key) or the legacy +// dockercfg type (a bare registry -> auth map). +func registryAuths(s *corev1.Secret) (map[string]json.RawMessage, error) { + if raw, ok := s.Data[corev1.DockerConfigJsonKey]; ok { + var cfg struct { + Auths map[string]json.RawMessage `json:"auths"` + } + if err := json.Unmarshal(raw, &cfg); err != nil { + return nil, err + } + if len(cfg.Auths) == 0 { + return nil, fmt.Errorf("secret %s has no auths entries", s.Name) + } + return cfg.Auths, nil + } + if raw, ok := s.Data[corev1.DockerConfigKey]; ok { + var auths map[string]json.RawMessage + if err := json.Unmarshal(raw, &auths); err != nil { + return nil, err + } + if len(auths) == 0 { + return nil, fmt.Errorf("secret %s has no auths entries", s.Name) + } + return auths, nil + } + return nil, fmt.Errorf("secret %s has neither a %s nor a %s key", s.Name, corev1.DockerConfigJsonKey, corev1.DockerConfigKey) +} diff --git a/pkg/enterprise/kubecontrollers_test.go b/pkg/enterprise/kubecontrollers_test.go index 41ee40ca72..3bb93c0e62 100644 --- a/pkg/enterprise/kubecontrollers_test.go +++ b/pkg/enterprise/kubecontrollers_test.go @@ -24,6 +24,7 @@ import ( v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -31,11 +32,15 @@ import ( operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/apis" "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/controller/certificatemanager" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/enterprise" "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/render/applicationlayer" rmeta "github.com/tigera/operator/pkg/render/common/meta" + "github.com/tigera/operator/pkg/render/common/networkpolicy" "github.com/tigera/operator/pkg/render/kubecontrollers" "github.com/tigera/operator/pkg/render/monitor" "github.com/tigera/operator/pkg/tls" @@ -122,3 +127,166 @@ func certManagementControllerContext() extensions.ControllerContext { CertificateManager: certManager, } } + +var _ = Describe("calico-kube-controllers enterprise surface", func() { + calicoKubeControllersCfg := func(cc extensions.ControllerContext) *kubecontrollers.KubeControllersConfiguration { + return &kubecontrollers.KubeControllersConfiguration{ + Installation: cc.Installation, + ClusterDomain: cc.ClusterDomain, + TrustedBundle: cc.TrustedBundle, + MetricsPort: 9094, + Namespace: common.CalicoNamespace, + BindingNamespaces: []string{common.CalicoNamespace}, + } + } + + // render builds the base calico-kube-controllers objects and applies the + // enterprise modifier, exactly as the component handler does. + renderKubeControllers := func(cc extensions.ControllerContext, rc extensions.RenderContext) []client.Object { + comp := kubecontrollers.NewCalicoKubeControllers(calicoKubeControllersCfg(cc)) + Expect(comp.ResolveImages(nil)).NotTo(HaveOccurred()) + create, del := comp.Objects() + out, _ := applyExtensions(ext, render.ComponentNameKubeControllers, rc, create, del) + return out + } + + kubeContainer := func(objs []client.Object) *corev1.Container { + dp, ok := extensions.FindObject[*appsv1.Deployment](objs, kubecontrollers.KubeController) + Expect(ok).To(BeTrue()) + return &dp.Spec.Template.Spec.Containers[0] + } + + It("layers the enterprise rules, controllers, and metrics TLS on (WAF off)", func() { + rc, _, err := ext.ExtendContext(newControllerContext(operatorv1.CalicoEnterprise)) + Expect(err).NotTo(HaveOccurred()) + objs := renderKubeControllers(newControllerContext(operatorv1.CalicoEnterprise), rc) + + role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, kubecontrollers.KubeControllerRole) + Expect(ok).To(BeTrue()) + Expect(role.Rules).To(ContainElement(HaveField("Resources", ContainElement("licensekeys")))) + + c := kubeContainer(objs) + Expect(c.Env).To(ContainElement(corev1.EnvVar{ + Name: "ENABLED_CONTROLLERS", Value: "node,loadbalancer,service,federatedservices,usage", + })) + // Metrics serving TLS wired from the keypair the hook created. + Expect(c.Env).To(ContainElement(HaveField("Name", "TLS_KEY_PATH"))) + // WAF is off, so no WASM env and no webhook objects. + Expect(c.Env).NotTo(ContainElement(HaveField("Name", "WASM_IMAGE"))) + _, ok = extensions.FindObject[*corev1.Service](objs, applicationlayer.WAFWebhookServiceName) + Expect(ok).To(BeFalse()) + }) + + It("layers the full WAF surface on when the GatewayAPI extension is enabled", func() { + cc := wafControllerContext() + rc, managed, err := ext.ExtendContext(cc) + Expect(err).NotTo(HaveOccurred()) + names := []string{} + for _, kp := range managed { + names = append(names, kp.GetName()) + } + Expect(names).To(ContainElement(applicationlayer.WAFWebhookServerTLSSecretName)) + + objs := renderKubeControllers(cc, rc) + + role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, kubecontrollers.KubeControllerRole) + Expect(ok).To(BeTrue()) + Expect(role.Rules).To(ContainElement(HaveField("Resources", ContainElement("wafpolicies")))) + + c := kubeContainer(objs) + Expect(c.Env).To(ContainElement(corev1.EnvVar{ + Name: "ENABLED_CONTROLLERS", Value: "node,loadbalancer,service,federatedservices,usage,applicationlayer", + })) + Expect(c.Env).To(ContainElement(corev1.EnvVar{ + Name: "WASM_IMAGE", Value: "test-reg/tigera/envoy-proxy:" + components.ComponentGatewayAPIEnvoyProxy.Version, + })) + Expect(c.Env).To(ContainElement(corev1.EnvVar{Name: "WASM_PULL_SECRET", Value: enterprise.WASMPullSecretName})) + Expect(c.Env).To(ContainElement(corev1.EnvVar{Name: "WASM_CA_CERT", Value: enterprise.WASMCACertName})) + Expect(c.Env).To(ContainElement(HaveField("Name", "WAF_WEBHOOK_CERT_DIR"))) + Expect(c.Ports).To(ContainElement(corev1.ContainerPort{Name: "waf-webhook", ContainerPort: int32(9443), Protocol: corev1.ProtocolTCP})) + + // The webhook surface, the wasm pull secret, and the wasm CA bundle are rendered. + _, ok = extensions.FindObject[*corev1.Service](objs, applicationlayer.WAFWebhookServiceName) + Expect(ok).To(BeTrue()) + _, ok = extensions.FindObject[*corev1.Secret](objs, enterprise.WASMPullSecretName) + Expect(ok).To(BeTrue()) + _, ok = extensions.FindObject[*corev1.ConfigMap](objs, enterprise.WASMCACertName) + Expect(ok).To(BeTrue()) + }) + + It("deletes the WAF webhook surface when the extension is disabled", func() { + cc := newControllerContext(operatorv1.CalicoEnterprise) + rc, _, err := ext.ExtendContext(cc) + Expect(err).NotTo(HaveOccurred()) + + comp := kubecontrollers.NewCalicoKubeControllers(calicoKubeControllersCfg(cc)) + Expect(comp.ResolveImages(nil)).NotTo(HaveOccurred()) + create, del := comp.Objects() + _, toDelete := applyExtensions(ext, render.ComponentNameKubeControllers, rc, create, del) + + _, ok := extensions.FindObject[*corev1.Service](toDelete, applicationlayer.WAFWebhookServiceName) + Expect(ok).To(BeTrue(), "the webhook Service should be queued for deletion") + }) + + It("adds the WAF webhook ingress rule to the network policy when enabled", func() { + cc := wafControllerContext() + rc, _, err := ext.ExtendContext(cc) + Expect(err).NotTo(HaveOccurred()) + + comp := kubecontrollers.NewCalicoKubeControllersPolicy(calicoKubeControllersCfg(cc), nil) + create, del := comp.Objects() + objs, _ := applyExtensions(ext, render.ComponentNameKubeControllersPolicy, rc, create, del) + + policy, ok := extensions.FindObject[*v3.NetworkPolicy](objs, kubecontrollers.KubeControllerNetworkPolicyName) + Expect(ok).To(BeTrue()) + Expect(policy.Spec.Ingress).To(ContainElement(v3.Rule{ + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: v3.EntityRule{ + Ports: networkpolicy.Ports(uint16(applicationlayer.WAFWebhookContainerPort)), + }, + })) + }) +}) + +// wafControllerContext builds a controller context with a WAF-enabled GatewayAPI CR +// and an install pull secret, so the installation hook produces the full WAF data. +func wafControllerContext() extensions.ControllerContext { + scheme := runtime.NewScheme() + Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) + c := ctrlrfake.DefaultFakeClientBuilder(scheme).Build() + + Expect(c.Create(context.Background(), &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "pull", Namespace: common.OperatorNamespace()}, + Type: corev1.SecretTypeDockerConfigJson, + Data: map[string][]byte{corev1.DockerConfigJsonKey: []byte(`{"auths":{"reg.example.com":{"auth":"abc"}}}`)}, + })).NotTo(HaveOccurred()) + + enabled := operatorv1.WAFExtensionStateEnabled + Expect(c.Create(context.Background(), &operatorv1.GatewayAPI{ + ObjectMeta: metav1.ObjectMeta{Name: "default"}, + Spec: operatorv1.GatewayAPISpec{ + Extensions: &operatorv1.GatewayAPIExtensions{WAF: &operatorv1.WAFExtensionSpec{State: &enabled}}, + }, + })).NotTo(HaveOccurred()) + + certManager, err := certificatemanager.Create(c, nil, "", common.OperatorNamespace(), certificatemanager.AllowCACreation()) + Expect(err).NotTo(HaveOccurred()) + + return extensions.ControllerContext{ + RenderContext: extensions.RenderContext{ + Installation: &operatorv1.InstallationSpec{ + Variant: operatorv1.CalicoEnterprise, + Registry: "test-reg/", + ImagePullSecrets: []corev1.LocalObjectReference{{Name: "pull"}}, + }, + FelixConfiguration: &v3.FelixConfiguration{}, + TrustedBundle: certManager.CreateTrustedBundle(), + ClusterDomain: "cluster.local", + }, + Controller: extensions.InstallationController, + Ctx: context.Background(), + Client: c, + CertificateManager: certManager, + } +} diff --git a/pkg/render/kubecontrollers/waf_pull_secret_test.go b/pkg/enterprise/waf_pull_secret_test.go similarity index 88% rename from pkg/render/kubecontrollers/waf_pull_secret_test.go rename to pkg/enterprise/waf_pull_secret_test.go index 793374f169..92cab8df9d 100644 --- a/pkg/render/kubecontrollers/waf_pull_secret_test.go +++ b/pkg/enterprise/waf_pull_secret_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package kubecontrollers_test +package enterprise_test import ( "encoding/json" @@ -22,7 +22,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/tigera/operator/pkg/common" - "github.com/tigera/operator/pkg/render/kubecontrollers" + "github.com/tigera/operator/pkg/enterprise" ) func dockerConfigJSONSecret(name string, auths map[string]any) *corev1.Secret { @@ -49,7 +49,7 @@ func mergedAuths(t *testing.T, s *corev1.Secret) map[string]map[string]string { } func TestMergeWAFPullSecret_MergesDisjointRegistries(t *testing.T) { - merged, skipped := kubecontrollers.MergeWAFPullSecret([]*corev1.Secret{ + merged, skipped := enterprise.MergeWAFPullSecret([]*corev1.Secret{ dockerConfigJSONSecret("tigera-pull-secret", map[string]any{"quay.io": map[string]string{"auth": "dGlnZXJh"}}), dockerConfigJSONSecret("mirror-pull-secret", map[string]any{"registry.example.com": map[string]string{"auth": "bWlycm9y"}}), }) @@ -59,7 +59,7 @@ func TestMergeWAFPullSecret_MergesDisjointRegistries(t *testing.T) { if merged == nil { t.Fatal("expected a merged secret") } - if merged.Name != kubecontrollers.WASMPullSecretName || merged.Namespace != common.CalicoNamespace { + if merged.Name != enterprise.WASMPullSecretName || merged.Namespace != common.CalicoNamespace { t.Fatalf("unexpected name/namespace: %s/%s", merged.Namespace, merged.Name) } if merged.Type != corev1.SecretTypeDockerConfigJson { @@ -72,7 +72,7 @@ func TestMergeWAFPullSecret_MergesDisjointRegistries(t *testing.T) { } func TestMergeWAFPullSecret_FirstSecretWinsOnDuplicateRegistry(t *testing.T) { - merged, _ := kubecontrollers.MergeWAFPullSecret([]*corev1.Secret{ + merged, _ := enterprise.MergeWAFPullSecret([]*corev1.Secret{ dockerConfigJSONSecret("first", map[string]any{"quay.io": map[string]string{"auth": "Zmlyc3Q="}}), dockerConfigJSONSecret("second", map[string]any{"quay.io": map[string]string{"auth": "c2Vjb25k"}}), }) @@ -88,7 +88,7 @@ func TestMergeWAFPullSecret_SkipsUnparseableSecrets(t *testing.T) { Type: corev1.SecretTypeDockerConfigJson, Data: map[string][]byte{corev1.DockerConfigJsonKey: []byte("not-json")}, } - merged, skipped := kubecontrollers.MergeWAFPullSecret([]*corev1.Secret{ + merged, skipped := enterprise.MergeWAFPullSecret([]*corev1.Secret{ bad, dockerConfigJSONSecret("good", map[string]any{"quay.io": map[string]string{"auth": "Z29vZA=="}}), }) @@ -111,7 +111,7 @@ func TestMergeWAFPullSecret_LegacyDockercfg(t *testing.T) { Type: corev1.SecretTypeDockercfg, Data: map[string][]byte{corev1.DockerConfigKey: cfg}, } - merged, skipped := kubecontrollers.MergeWAFPullSecret([]*corev1.Secret{legacy}) + merged, skipped := enterprise.MergeWAFPullSecret([]*corev1.Secret{legacy}) if len(skipped) != 0 { t.Fatalf("expected no skipped secrets, got %v", skipped) } @@ -127,7 +127,7 @@ func TestMergeWAFPullSecret_NothingUsableReturnsNil(t *testing.T) { Type: corev1.SecretTypeDockerConfigJson, Data: map[string][]byte{corev1.DockerConfigJsonKey: []byte("not-json")}, } - merged, skipped := kubecontrollers.MergeWAFPullSecret([]*corev1.Secret{bad}) + merged, skipped := enterprise.MergeWAFPullSecret([]*corev1.Secret{bad}) if merged != nil { t.Fatalf("expected nil secret, got %v", merged) } @@ -141,8 +141,8 @@ func TestMergeWAFPullSecret_DeterministicOutput(t *testing.T) { dockerConfigJSONSecret("a", map[string]any{"z.example.com": map[string]string{"auth": "eg=="}, "a.example.com": map[string]string{"auth": "YQ=="}}), dockerConfigJSONSecret("b", map[string]any{"m.example.com": map[string]string{"auth": "bQ=="}}), } - first, _ := kubecontrollers.MergeWAFPullSecret(in) - second, _ := kubecontrollers.MergeWAFPullSecret(in) + first, _ := enterprise.MergeWAFPullSecret(in) + second, _ := enterprise.MergeWAFPullSecret(in) if string(first.Data[corev1.DockerConfigJsonKey]) != string(second.Data[corev1.DockerConfigJsonKey]) { t.Fatal("merged secret bytes must be deterministic across reconciles") } diff --git a/pkg/render/component.go b/pkg/render/component.go index 6fdf85303a..93e4824f3a 100644 --- a/pkg/render/component.go +++ b/pkg/render/component.go @@ -81,4 +81,8 @@ const ( // es-calico-kube-controllers deployment shares the component type but leaves // its modifier key empty, so it is not decorated. ComponentNameKubeControllers = "kube-controllers" + + // ComponentNameKubeControllersPolicy keys the calico-kube-controllers network + // policy modifier (the WAF admission webhook ingress rule). + ComponentNameKubeControllersPolicy = "kube-controllers-policy" ) diff --git a/pkg/render/kubecontrollers/kube-controllers.go b/pkg/render/kubecontrollers/kube-controllers.go index 46dbf2cd8a..36bc0068da 100644 --- a/pkg/render/kubecontrollers/kube-controllers.go +++ b/pkg/render/kubecontrollers/kube-controllers.go @@ -16,7 +16,6 @@ package kubecontrollers import ( "fmt" - "path/filepath" "slices" "strconv" "strings" @@ -36,7 +35,6 @@ import ( "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/controller/k8sapi" "github.com/tigera/operator/pkg/render" - "github.com/tigera/operator/pkg/render/applicationlayer" rcomp "github.com/tigera/operator/pkg/render/common/components" rmeta "github.com/tigera/operator/pkg/render/common/meta" "github.com/tigera/operator/pkg/render/common/networkpolicy" @@ -54,21 +52,6 @@ const ( KubeControllerMetrics = "calico-kube-controllers-metrics" KubeControllerNetworkPolicyName = networkpolicy.CalicoComponentPolicyPrefix + "kube-controller-access" - // WASMPullSecretName is the dedicated image-pull Secret (a renamed copy of - // the install pull secret) that the WAF reconciler replicates into tenant - // namespaces for the Coraza wasm OCI pull. A dedicated name avoids clashing - // with the operator-managed tigera-pull-secret the GatewayAPI render also - // copies into those namespaces (EV-6386). - WASMPullSecretName = "tigera-waf-pull-secret" - - // WASMCACertName is the dedicated CA-bundle ConfigMap (in the controller - // namespace) the WAF reconciler replicates into tenant namespaces for the - // Coraza wasm OCI registry TLS check — a dedicated name avoids clashing with - // the operator-managed tigera-ca-bundle ConfigMap the GatewayAPI render also - // copies there (EV-6386). The source copy is a renamed copy of the trusted - // bundle, provisioned by the core controller and passed in as WASMCACert. - WASMCACertName = "tigera-waf-ca-bundle" - // ManagedClustersWatchRoleBindingName binds kube-controllers to the managed-cluster // watch ClusterRole. Used by both calico-kube-controllers (in a management cluster) // and the enterprise es-calico-kube-controllers, so the binding stays generic here. @@ -100,8 +83,6 @@ type KubeControllersConfiguration struct { // namespace to be returned by the rendered. Expected that the calling code // take care to pass the same secret on each reconcile where possible. KubeControllersGatewaySecret *corev1.Secret - WASMPullSecret *corev1.Secret - WASMCACert *corev1.ConfigMap TrustedBundle certificatemanagement.TrustedBundleRO // Namespace to be installed into. @@ -114,29 +95,6 @@ type KubeControllersConfiguration struct { // If this is nil, then we should run in zero-tenant mode. Tenant *operatorv1.Tenant - // WAFGatewayExtensionEnabled gates the WAF v3 (Gateway API add-on) surface - // on calico-kube-controllers: the applicationlayer controller enablement, - // the WAF / Gateway-API / EnvoyExtensionPolicy / event / secret-replication - // RBAC, the WASM_IMAGE / WASM_PULL_SECRET / WASM_CA_CERT env vars, and the - // gateway envoy-proxy wasm image resolution. Sourced from - // `GatewayAPI.spec.extensions.waf.state == Enabled` (default off). - // See design `tigera/designs#25` (PMREQ-384). - WAFGatewayExtensionEnabled bool - - // WAFWebhookServerTLS is the serving certificate for the in-process WAF - // SecLang validating admission webhook hosted by calico-kube-controllers. - // When set (WAF enabled), it is mounted into the Pod and the webhook server - // reads it from WAF_WEBHOOK_CERT_DIR. Issued for the tigera-waf-webhook - // Service DNS name. Nil leaves the Deployment untouched (and the in-process - // server self-disables when the cert is absent). - WAFWebhookServerTLS certificatemanagement.KeyPairInterface - - // WAFWebhookCABundle is the PEM of the CA that issued WAFWebhookServerTLS - // (the operator CA), stamped into the ValidatingWebhookConfiguration's - // caBundle so the apiserver can verify the in-process webhook endpoint. - // Only consulted when WAFGatewayExtensionEnabled is true. - WAFWebhookCABundle []byte - // The fields below parameterize the generic kube-controllers component. The // variant assemblers (NewCalicoKubeControllers, the enterprise es builder) // fill them; the component renders them without any variant or component-name @@ -165,18 +123,25 @@ type KubeControllersConfiguration struct { ExtraEnv []corev1.EnvVar // DisableConfigAPI sets DISABLE_KUBE_CONTROLLERS_CONFIG_API. DisableConfigAPI bool - // ManageWAFWebhook makes this component own the in-process WAF admission - // webhook surface lifecycle (rendered when WAFGatewayExtensionEnabled, deleted - // otherwise). Only the calico-kube-controllers component sets this. - ManageWAFWebhook bool // ModifierKey is the extension modifier key the component reports through // render.Extensible. calico-kube-controllers sets it so the enterprise modifier - // can layer on its metrics TLS; es-calico-kube-controllers leaves it empty so it - // is never decorated. + // can layer on the enterprise surface; es-calico-kube-controllers leaves it empty + // so it is never decorated. ModifierKey string } +// calicoKubeControllersPolicyComponent wraps the calico-kube-controllers network +// policy passthrough so it is render.Extensible: the enterprise modifier adds the WAF +// admission webhook ingress rule. The base policy carries no WAF. +type calicoKubeControllersPolicyComponent struct { + render.Component +} + +func (calicoKubeControllersPolicyComponent) ModifierKey() string { + return render.ComponentNameKubeControllersPolicy +} + func NewCalicoKubeControllersPolicy(cfg *KubeControllersConfiguration, defaultDeny *v3.NetworkPolicy) render.Component { toCreate := []client.Object{kubeControllersCalicoSystemPolicy(cfg)} @@ -184,14 +149,14 @@ func NewCalicoKubeControllersPolicy(cfg *KubeControllersConfiguration, defaultDe toCreate = append(toCreate, defaultDeny) } - return render.NewPassthrough( + return calicoKubeControllersPolicyComponent{render.NewPassthrough( toCreate, []client.Object{ // allow-tigera Tier was renamed to calico-system networkpolicy.DeprecatedAllowTigeraNetworkPolicyObject("kube-controller-access", cfg.Namespace), networkpolicy.DeprecatedAllowTigeraNetworkPolicyObject("default-deny", common.CalicoNamespace), }, - ) + )} } // NewKubeControllers builds a kube-controllers component from a fully-populated @@ -202,80 +167,30 @@ func NewKubeControllers(cfg *KubeControllersConfiguration) render.Component { return &kubeControllersComponent{cfg: cfg} } +// NewCalicoKubeControllers builds the calico-kube-controllers component. The base is +// pure OSS (the common rules plus the node and loadbalancer controllers); the Calico +// Enterprise additions (extra RBAC, enterprise controllers, metrics TLS, the WAF v3 +// surface) are layered on by the enterprise modifier keyed by ModifierKey. func NewCalicoKubeControllers(cfg *KubeControllersConfiguration) render.Component { cfg.Name = KubeController cfg.ConfigName = "default" cfg.RoleName = KubeControllerRole cfg.RoleBindingName = KubeControllerRoleBinding cfg.MetricsName = KubeControllerMetrics - cfg.ManageWAFWebhook = true cfg.ModifierKey = render.ComponentNameKubeControllers cfg.Rules = KubeControllersRoleCommonRules(cfg) cfg.EnabledControllers = []string{"node", "loadbalancer"} - if cfg.Installation.Variant.IsEnterprise() { - cfg.Rules = append(cfg.Rules, KubeControllersRoleEnterpriseCommonRules(cfg)...) - cfg.Rules = append(cfg.Rules, - rbacv1.PolicyRule{ - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"remoteclusterconfigurations"}, - Verbs: []string{"watch", "list", "get"}, - }, - rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"endpoints"}, - Verbs: []string{"create", "update", "delete"}, - }, - rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"namespaces"}, - Verbs: []string{"get"}, - }, - rbacv1.PolicyRule{ - APIGroups: []string{"usage.tigera.io"}, - Resources: []string{"licenseusagereports"}, - Verbs: []string{"create", "update", "delete", "watch", "list", "get"}, - }, - ) - cfg.EnabledControllers = append(cfg.EnabledControllers, "service", "federatedservices", "usage") - if cfg.WAFGatewayExtensionEnabled { - cfg.EnabledControllers = append(cfg.EnabledControllers, "applicationlayer") - } - cfg.ExtraEnv = calicoEnterpriseEnv(cfg) - } return NewKubeControllers(cfg) } -// calicoEnterpriseEnv builds the enterprise-only static env vars for -// calico-kube-controllers. The dynamic WASM_* vars depend on resolved images and -// are added at deployment-render time. -func calicoEnterpriseEnv(cfg *KubeControllersConfiguration) []corev1.EnvVar { - var env []corev1.EnvVar - if cfg.Tenant != nil { - env = append(env, corev1.EnvVar{Name: "TENANT_ID", Value: cfg.Tenant.Spec.ID}) - } - if cfg.TrustedBundle != nil { - env = append(env, corev1.EnvVar{Name: "MULTI_CLUSTER_FORWARDING_CA", Value: cfg.TrustedBundle.MountPath()}) - } - if cfg.Installation.CalicoNetwork != nil && cfg.Installation.CalicoNetwork.MultiInterfaceMode != nil { - env = append(env, corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: cfg.Installation.CalicoNetwork.MultiInterfaceMode.Value()}) - } - return env -} - type kubeControllersComponent struct { // cfg is caller-supplied configuration for building kube-controllers Kubernetes resources. cfg *KubeControllersConfiguration // Internal state generated by the given configuration. calicoImage string - - // wasmImage is the fully-resolved OCI reference for the Coraza WAF wasm - // binary (Enterprise only). Surfaced to the kube-controllers binary via - // the WASM_IMAGE env var; consumed by the applicationlayer reconcilers - // in tigera/calico-private to program WAF policy attachments. - wasmImage string } func (c *kubeControllersComponent) ResolveImages(is *operatorv1.ImageSet) error { @@ -287,15 +202,6 @@ func (c *kubeControllersComponent) ResolveImages(is *operatorv1.ImageSet) error if err != nil { return err } - if c.cfg.WAFGatewayExtensionEnabled { - // The Coraza WAF wasm is baked into the gateway envoy-proxy image as its - // final layer; Envoy Gateway extracts it from there. Point WASM_IMAGE at - // that same image (no standalone coraza-wasm image needed). - c.wasmImage, err = components.GetReference(components.ComponentGatewayAPIEnvoyProxy, reg, path, prefix, is) - if err != nil { - return err - } - } return nil } @@ -339,26 +245,6 @@ func (c *kubeControllersComponent) Objects() ([]client.Object, []client.Object) objectsToCreate = append(objectsToCreate, secret.ToRuntimeObjects( secret.CopyToNamespace(c.cfg.Namespace, c.cfg.KubeControllersGatewaySecret)...)...) } - if c.cfg.WASMPullSecret != nil { - objectsToCreate = append(objectsToCreate, secret.ToRuntimeObjects( - secret.CopyToNamespace(c.cfg.Namespace, c.cfg.WASMPullSecret)...)...) - } - if c.cfg.WASMCACert != nil { - objectsToCreate = append(objectsToCreate, c.cfg.WASMCACert) - } - - // The in-process WAF admission webhook surface (Service fronting this Pod + - // ValidatingWebhookConfiguration). Rendered here, rather than as a - // passthrough in the core controller, so the objects are cleaned up when the - // WAF extension is disabled or the GatewayAPI CR is removed. - if c.cfg.ManageWAFWebhook { - webhookObjs := applicationlayer.WAFAdmissionWebhookComponents(c.cfg.WAFWebhookCABundle) - if c.cfg.WAFGatewayExtensionEnabled { - objectsToCreate = append(objectsToCreate, webhookObjs...) - } else { - objectsToDelete = append(objectsToDelete, webhookObjs...) - } - } if c.cfg.MetricsPort != 0 { objectsToCreate = append(objectsToCreate, c.prometheusService()) @@ -510,153 +396,6 @@ func KubeControllersRoleCommonRules(cfg *KubeControllersConfiguration) []rbacv1. return rules } -func KubeControllersRoleEnterpriseCommonRules(cfg *KubeControllersConfiguration) []rbacv1.PolicyRule { - rules := []rbacv1.PolicyRule{ - { - APIGroups: []string{""}, - Resources: []string{"configmaps"}, - Verbs: []string{"watch", "list", "get", "update", "create", "delete"}, - }, - { - // The Federated Services Controller needs access to the remote kubeconfig secret - // in order to create a remote syncer. - APIGroups: []string{""}, - Resources: []string{"secrets"}, - Verbs: []string{"watch", "list", "get"}, - }, - { - // Needed to validate the license - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"licensekeys"}, - Verbs: []string{"get", "watch", "list"}, - }, - { - // Needed to update the status of the LicenseKey with the result of license validation. - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"licensekeys/status"}, - Verbs: []string{"update"}, - }, - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"deeppacketinspections"}, - Verbs: []string{"get", "watch", "list"}, - }, - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"deeppacketinspections/status"}, - Verbs: []string{"update"}, - }, - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"packetcaptures"}, - Verbs: []string{"get", "list", "update"}, - }, - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"packetcaptures/status"}, - Verbs: []string{"update"}, - }, - } - - if cfg.WAFGatewayExtensionEnabled { - // WAF v3 (Gateway API add-on) RBAC. Gated by - // GatewayAPI.spec.extensions.waf.state == Enabled. - rules = append(rules, - // Application-layer (gateway-addons) reconcilers reconcile WAF resources - // against Gateway API targetRefs and emit events on the policy objects. - rbacv1.PolicyRule{ - APIGroups: []string{"applicationlayer.projectcalico.org"}, - Resources: []string{ - "wafpolicies", "globalwafpolicies", - "wafplugins", "globalwafplugins", - "wafvalidationpolicies", "globalwafvalidationpolicies", - }, - Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, - }, - rbacv1.PolicyRule{ - APIGroups: []string{"applicationlayer.projectcalico.org"}, - Resources: []string{ - "wafpolicies/status", "globalwafpolicies/status", - "wafplugins/status", "globalwafplugins/status", - "wafvalidationpolicies/status", "globalwafvalidationpolicies/status", - }, - Verbs: []string{"get", "update", "patch"}, - }, - rbacv1.PolicyRule{ - APIGroups: []string{"applicationlayer.projectcalico.org"}, - Resources: []string{ - "wafpolicies/finalizers", "globalwafpolicies/finalizers", - "wafplugins/finalizers", "globalwafplugins/finalizers", - "wafvalidationpolicies/finalizers", "globalwafvalidationpolicies/finalizers", - }, - Verbs: []string{"update"}, - }, - rbacv1.PolicyRule{ - // Validate Gateway API targetRefs and surface attachment status. - APIGroups: []string{"gateway.networking.k8s.io"}, - Resources: []string{"gateways", "httproutes", "tcproutes", "tlsroutes", "grpcroutes"}, - Verbs: []string{"get", "list", "watch", "update", "patch"}, - }, - rbacv1.PolicyRule{ - APIGroups: []string{"gateway.networking.k8s.io"}, - Resources: []string{"gateways/status", "httproutes/status", "tcproutes/status", "tlsroutes/status", "grpcroutes/status"}, - Verbs: []string{"get", "update", "patch"}, - }, - // controller-runtime Reconcilers (e.g. the applicationlayer manager) record - // events on watched objects via Recorder.Eventf; both core and events.k8s.io - // API groups are emitted depending on the kubernetes version. - rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"events"}, - Verbs: []string{"create", "patch"}, - }, - rbacv1.PolicyRule{ - APIGroups: []string{"events.k8s.io"}, - Resources: []string{"events"}, - Verbs: []string{"create", "patch"}, - }, - // Application-layer reconciler replicates the WAF wasm pull Secret from - // the controller namespace (calico-system) into each WAFPolicy's - // namespace so the rendered EnvoyExtensionPolicy can reference it. Also - // replicates CA-cert ConfigMaps when WASM_CA_CERT is set. - rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"secrets", "configmaps"}, - Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, - }, - // Application-layer reconciler emits one EnvoyExtensionPolicy per WAF - // targetRef to bind the Coraza wasm filter at the gateway / route. - rbacv1.PolicyRule{ - APIGroups: []string{"gateway.envoyproxy.io"}, - Resources: []string{"envoyextensionpolicies"}, - Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, - }, - // Application-layer reconciler stamps each namespace with its - // allocated WAF rule-id range (applicationlayer.projectcalico.org/waf-id-range - // annotation) so application operators can author in-range rules. The - // base role already grants namespaces get/list/watch; the annotation - // write needs patch/update, gated to the WAF path. - rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"namespaces"}, - Verbs: []string{"get", "patch", "update"}, - }, - ) - } - - if cfg.ManagementClusterConnection != nil { - rules = append(rules, - rbacv1.PolicyRule{ - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"licensekeys"}, - Verbs: []string{"get", "create", "update", "list", "watch"}, - }, - ) - } - - return rules -} - func (c *kubeControllersComponent) controllersServiceAccount() *corev1.ServiceAccount { return &corev1.ServiceAccount{ TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}, @@ -715,54 +454,11 @@ func (c *kubeControllersComponent) controllersDeployment() *appsv1.Deployment { env = append(env, c.cfg.K8sServiceEpPodNetwork.EnvVars()...) env = append(env, c.cfg.ExtraEnv...) - // Application-layer (gateway-addons / WAF v3) env vars, gated by - // GatewayAPI.spec.extensions.waf.state == Enabled. When the gate is off - // (default), none of the WASM_* env vars are rendered and the kube-controllers - // binary skips the WAF reconcilers entirely (see the applicationlayer entry in - // EnabledControllers). The WASM_IMAGE value depends on resolved images, so this - // is rendered here rather than in the static ExtraEnv. - if c.cfg.WAFGatewayExtensionEnabled { - // Application-layer (gateway-addons) reconcilers consume the Coraza WAF - // wasm OCI reference from this env var to program WAF policy attachments. - // Empty when ResolveImages was not called for the Calico variant; the - // reconciler stamps Programmed=False/WASMUnavailable in that case. - if c.wasmImage != "" { - env = append(env, corev1.EnvVar{Name: "WASM_IMAGE", Value: c.wasmImage}) - } - - // WASM_PULL_SECRET names the imagePullSecret the reconciler replicates - // from the kube-controllers namespace into a WAFPolicy's namespace so - // the rendered EnvoyExtensionPolicy can pull the wasm OCI artifact from - // a private Tigera registry. Source the name from the first - // Installation.ImagePullSecrets entry so multi-tenant / BYO-registry - // installs reuse whatever pull secret operator already attaches here. - if c.cfg.WASMPullSecret != nil { - env = append(env, corev1.EnvVar{Name: "WASM_PULL_SECRET", Value: c.cfg.WASMPullSecret.Name}) - } - - // WASM_CA_CERT names the dedicated CA bundle ConfigMap (provisioned as - // WASMCACert) that the reconciler replicates alongside WASM_PULL_SECRET - // so the EnvoyExtensionPolicy wasm fetcher trusts the registry's TLS - // chain. Only set when the source ConfigMap is actually rendered. - if c.cfg.WASMCACert != nil { - env = append(env, corev1.EnvVar{Name: "WASM_CA_CERT", Value: c.cfg.WASMCACert.Name}) - } - } - if c.cfg.TrustedBundle != nil { env = append(env, corev1.EnvVar{Name: "CA_CRT_PATH", Value: c.cfg.TrustedBundle.MountPath()}, ) } - if c.cfg.WAFWebhookServerTLS != nil { - // The in-process WAF admission webhook server (calico-private - // applicationlayer manager) reads its serving cert (tls.crt/tls.key) - // from this directory; the controller-runtime webhook server only - // registers when the cert is present. - env = append(env, - corev1.EnvVar{Name: "WAF_WEBHOOK_CERT_DIR", Value: filepath.Dir(c.cfg.WAFWebhookServerTLS.VolumeMountCertificateFilePath())}, - ) - } // UID 999 is used in kube-controller Dockerfile. sc := securitycontext.NewNonRootContext() @@ -806,20 +502,7 @@ func (c *kubeControllersComponent) controllersDeployment() *appsv1.Deployment { VolumeMounts: c.kubeControllersVolumeMounts(), } - if c.cfg.WAFWebhookServerTLS != nil { - // Expose the in-process WAF admission-webhook port that the - // tigera-waf-webhook Service forwards to. - container.Ports = append(container.Ports, corev1.ContainerPort{ - Name: "waf-webhook", - ContainerPort: applicationlayer.WAFWebhookContainerPort, - Protocol: corev1.ProtocolTCP, - }) - } - var initContainers []corev1.Container - if c.cfg.WAFWebhookServerTLS != nil && c.cfg.WAFWebhookServerTLS.UseCertificateManagement() { - initContainers = append(initContainers, c.cfg.WAFWebhookServerTLS.InitContainer(c.cfg.Namespace, sc)) - } tolerations := appendUniqueTolerations(c.cfg.Installation.ControlPlaneTolerations, rmeta.TolerateCriticalAddonsAndControlPlane...) if c.cfg.Installation.KubernetesProvider.IsGKE() { tolerations = appendUniqueTolerations(tolerations, rmeta.TolerateGKEARM64NoSchedule) @@ -964,9 +647,6 @@ func (c *kubeControllersComponent) kubeControllersVolumeMounts() []corev1.Volume if c.cfg.TrustedBundle != nil { mounts = append(mounts, c.cfg.TrustedBundle.VolumeMounts(c.SupportedOSType())...) } - if c.cfg.WAFWebhookServerTLS != nil { - mounts = append(mounts, c.cfg.WAFWebhookServerTLS.VolumeMount(c.SupportedOSType())) - } return mounts } @@ -975,9 +655,6 @@ func (c *kubeControllersComponent) kubeControllersVolumes() []corev1.Volume { if c.cfg.TrustedBundle != nil { volumes = append(volumes, c.cfg.TrustedBundle.Volume()) } - if c.cfg.WAFWebhookServerTLS != nil { - volumes = append(volumes, c.cfg.WAFWebhookServerTLS.Volume()) - } return volumes } @@ -1020,20 +697,6 @@ func kubeControllersCalicoSystemPolicy(cfg *KubeControllersConfiguration) *v3.Ne }) } - // Allow the kube-apiserver to reach the in-process WAF admission webhook on - // :9443 (EV-6386). render-v3 wires the webhook Service/config/cert + the - // server, but without this ingress rule the calico-system default-deny drops - // the apiserver→:9443 call and every WAFPolicy/WAFPlugin admission times out. - if cfg.WAFGatewayExtensionEnabled { - ingressRules = append(ingressRules, v3.Rule{ - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: v3.EntityRule{ - Ports: networkpolicy.Ports(uint16(applicationlayer.WAFWebhookContainerPort)), - }, - }) - } - if r, err := cfg.K8sServiceEp.DestinationEntityRule(); r != nil && err == nil { egressRules = append(egressRules, v3.Rule{ Action: v3.Allow, diff --git a/pkg/render/kubecontrollers/kube-controllers_test.go b/pkg/render/kubecontrollers/kube-controllers_test.go index 801c4b2687..89cfad0940 100644 --- a/pkg/render/kubecontrollers/kube-controllers_test.go +++ b/pkg/render/kubecontrollers/kube-controllers_test.go @@ -16,12 +16,10 @@ package kubecontrollers_test import ( "fmt" - "path/filepath" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - admissionregistrationv1 "k8s.io/api/admissionregistration/v1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -43,7 +41,6 @@ import ( "github.com/tigera/operator/pkg/dns" "github.com/tigera/operator/pkg/enterprise" "github.com/tigera/operator/pkg/render" - "github.com/tigera/operator/pkg/render/applicationlayer" rmeta "github.com/tigera/operator/pkg/render/common/meta" "github.com/tigera/operator/pkg/render/common/networkpolicy" rtest "github.com/tigera/operator/pkg/render/common/test" @@ -231,175 +228,6 @@ var _ = Describe("kube-controllers rendering tests", func() { Expect(ds.Spec.Template.Spec.Tolerations).To(ConsistOf(rmeta.TolerateCriticalAddonsAndControlPlane)) }) - It("should render all calico kube-controllers resources for a default configuration (standalone) using CalicoEnterprise", func() { - expectedResources := []struct { - name string - ns string - group string - version string - kind string - }{ - {name: kubecontrollers.KubeControllerServiceAccount, ns: common.CalicoNamespace, group: "", version: "v1", kind: "ServiceAccount"}, - {name: kubecontrollers.KubeControllerRole, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, - {name: kubecontrollers.KubeControllerRoleBinding, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: kubecontrollers.KubeController, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "Deployment"}, - {name: kubecontrollers.WASMPullSecretName, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Secret"}, - {name: kubecontrollers.WASMCACertName, ns: common.CalicoNamespace, group: "", version: "v1", kind: "ConfigMap"}, - {name: applicationlayer.WAFWebhookServiceName, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Service"}, - {name: "tigera-waf.applicationlayer.projectcalico.org", ns: "", group: "admissionregistration.k8s.io", version: "v1", kind: "ValidatingWebhookConfiguration"}, - {name: kubecontrollers.KubeControllerMetrics, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Service"}, - } - - instance.Variant = operatorv1.CalicoEnterprise - instance.ImagePullSecrets = []corev1.LocalObjectReference{{Name: "tigera-pull-secret"}} - cfg.MetricsPort = 9094 - // Opt in to the WAF Gateway API add-on so the WAF env vars + RBAC are rendered. - cfg.WAFGatewayExtensionEnabled = true - cfg.WAFWebhookCABundle = []byte("fake-ca-bundle") - // core_controller provisions a dedicated WAF wasm pull secret (a renamed - // copy of the install pull secret) so the reconciler can replicate it into - // WAFPolicy namespaces without clashing with the operator-managed - // tigera-pull-secret; surface it here so it renders and WASM_PULL_SECRET is set. - cfg.WASMPullSecret = &corev1.Secret{TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: kubecontrollers.WASMPullSecretName, Namespace: common.CalicoNamespace}} - // Likewise core_controller provisions the dedicated WAF wasm CA-bundle - // ConfigMap (a renamed copy of the trusted bundle); surface it here so it - // renders and WASM_CA_CERT is set. - cfg.WASMCACert = &corev1.ConfigMap{TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: kubecontrollers.WASMCACertName, Namespace: common.CalicoNamespace}} - - component := kubecontrollers.NewCalicoKubeControllers(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - Expect(len(resources)).To(Equal(len(expectedResources))) - - // Should render the correct resources. - i := 0 - for _, expectedRes := range expectedResources { - rtest.ExpectResourceTypeAndObjectMetadata(resources[i], expectedRes.name, expectedRes.ns, expectedRes.group, expectedRes.version, expectedRes.kind) - i++ - } - - // The Deployment should have the correct configuration. - dp := rtest.GetResource(resources, kubecontrollers.KubeController, common.CalicoNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) - - Expect(dp.Spec.Template.Spec.Containers[0].Image).To(Equal("test-reg/tigera/calico:" + components.ComponentTigeraCalico.Version)) - Expect(dp.Spec.Template.Spec.ImagePullSecrets).To(ContainElement(corev1.LocalObjectReference{Name: "tigera-pull-secret"})) - envs := dp.Spec.Template.Spec.Containers[0].Env - Expect(envs).To(ContainElement(corev1.EnvVar{ - Name: "ENABLED_CONTROLLERS", Value: "node,loadbalancer,service,federatedservices,usage,applicationlayer", - })) - // Application-layer reconcilers consume these env vars to program WAF - // EnvoyExtensionPolicy attachments. - Expect(envs).To(ContainElement(corev1.EnvVar{ - Name: "WASM_IMAGE", Value: "test-reg/tigera/envoy-proxy:" + components.ComponentGatewayAPIEnvoyProxy.Version, - })) - Expect(envs).To(ContainElement(corev1.EnvVar{ - Name: "WASM_PULL_SECRET", Value: kubecontrollers.WASMPullSecretName, - })) - // WASM_CA_CERT names the dedicated WAF trusted-bundle ConfigMap that the - // reconciler replicates into WAFPolicy namespaces (kept separate from the - // operator-managed tigera-ca-bundle the GatewayAPI render also copies there). - Expect(envs).To(ContainElement(corev1.EnvVar{ - Name: "WASM_CA_CERT", Value: kubecontrollers.WASMCACertName, - })) - - Expect(len(dp.Spec.Template.Spec.Containers[0].VolumeMounts)).To(Equal(1)) - Expect(len(dp.Spec.Template.Spec.Volumes)).To(Equal(1)) - - clusterRole := rtest.GetResource(resources, kubecontrollers.KubeControllerRole, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - Expect(clusterRole.Rules).To(HaveLen(38), "cluster role should have 38 rules") - - // Application-layer reconciler RBAC: WAF CRDs (resources, /status, /finalizers). - Expect(clusterRole.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{"applicationlayer.projectcalico.org"}, - Resources: []string{ - "wafpolicies", "globalwafpolicies", - "wafplugins", "globalwafplugins", - "wafvalidationpolicies", "globalwafvalidationpolicies", - }, - Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, - })) - Expect(clusterRole.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{"applicationlayer.projectcalico.org"}, - Resources: []string{ - "wafpolicies/status", "globalwafpolicies/status", - "wafplugins/status", "globalwafplugins/status", - "wafvalidationpolicies/status", "globalwafvalidationpolicies/status", - }, - Verbs: []string{"get", "update", "patch"}, - })) - Expect(clusterRole.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{"applicationlayer.projectcalico.org"}, - Resources: []string{ - "wafpolicies/finalizers", "globalwafpolicies/finalizers", - "wafplugins/finalizers", "globalwafplugins/finalizers", - "wafvalidationpolicies/finalizers", "globalwafvalidationpolicies/finalizers", - }, - Verbs: []string{"update"}, - })) - // Gateway API targetRef validation + status patching. - Expect(clusterRole.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{"gateway.networking.k8s.io"}, - Resources: []string{"gateways", "httproutes", "tcproutes", "tlsroutes", "grpcroutes"}, - Verbs: []string{"get", "list", "watch", "update", "patch"}, - })) - Expect(clusterRole.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{"gateway.networking.k8s.io"}, - Resources: []string{"gateways/status", "httproutes/status", "tcproutes/status", "tlsroutes/status", "grpcroutes/status"}, - Verbs: []string{"get", "update", "patch"}, - })) - // Recorder.Eventf emits to both core/events and events.k8s.io/events. - Expect(clusterRole.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"events"}, - Verbs: []string{"create", "patch"}, - })) - Expect(clusterRole.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{"events.k8s.io"}, - Resources: []string{"events"}, - Verbs: []string{"create", "patch"}, - })) - // Cluster-wide secrets+configmaps CRUD: reconciler replicates pull - // secrets and CA bundles from the controller namespace into target - // WAFPolicy namespaces. - Expect(clusterRole.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"secrets", "configmaps"}, - Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, - })) - // EnvoyExtensionPolicy CRUD: reconciler renders one EEP per WAF targetRef. - Expect(clusterRole.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{"gateway.envoyproxy.io"}, - Resources: []string{"envoyextensionpolicies"}, - Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, - })) - - ms := rtest.GetResource(resources, kubecontrollers.KubeControllerMetrics, common.CalicoNamespace, "", "v1", "Service").(*corev1.Service) - Expect(ms.Spec.ClusterIP).To(Equal("None"), "metrics service should be headless") - - // The webhook surface is rendered with the operator CA stamped into the - // ValidatingWebhookConfiguration caBundle. - vwc := rtest.GetResource(resources, "tigera-waf.applicationlayer.projectcalico.org", "", "admissionregistration.k8s.io", "v1", "ValidatingWebhookConfiguration").(*admissionregistrationv1.ValidatingWebhookConfiguration) - Expect(vwc.Webhooks).To(HaveLen(1)) - Expect(vwc.Webhooks[0].ClientConfig.CABundle).To(Equal([]byte("fake-ca-bundle"))) - }) - - It("should delete the WAF admission webhook surface when the WAF Gateway API add-on is disabled", func() { - instance.Variant = operatorv1.CalicoEnterprise - cfg.WAFGatewayExtensionEnabled = false - - component := kubecontrollers.NewCalicoKubeControllers(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - toCreate, toDelete := component.Objects() - - // Neither webhook object is created... - Expect(rtest.GetResource(toCreate, applicationlayer.WAFWebhookServiceName, common.CalicoNamespace, "", "v1", "Service")).To(BeNil()) - Expect(rtest.GetResource(toCreate, "tigera-waf.applicationlayer.projectcalico.org", "", "admissionregistration.k8s.io", "v1", "ValidatingWebhookConfiguration")).To(BeNil()) - // ...and both are queued for deletion, so disabling the feature (or - // removing the GatewayAPI CR) cleans up an earlier enabled render. - Expect(rtest.GetResource(toDelete, applicationlayer.WAFWebhookServiceName, common.CalicoNamespace, "", "v1", "Service")).NotTo(BeNil()) - Expect(rtest.GetResource(toDelete, "tigera-waf.applicationlayer.projectcalico.org", "", "admissionregistration.k8s.io", "v1", "ValidatingWebhookConfiguration")).NotTo(BeNil()) - }) - It("should render all calico kube-controllers resources using CalicoEnterprise on Openshift", func() { expectedResources := []struct { name string @@ -448,8 +276,6 @@ var _ = Describe("kube-controllers rendering tests", func() { instance.Variant = operatorv1.CalicoEnterprise cfg.KubeControllersGatewaySecret = &testutils.KubeControllersUserSecret cfg.MetricsPort = 9094 - // Opt in to the WAF Gateway API add-on so the WAF env vars + RBAC are rendered. - cfg.WAFGatewayExtensionEnabled = true component := enterprise.NewElasticsearchKubeControllers(&cfg) Expect(component.ResolveImages(nil)).To(BeNil()) @@ -482,7 +308,7 @@ var _ = Describe("kube-controllers rendering tests", func() { Expect(dp.Spec.Template.Spec.Volumes[0].ConfigMap.Name).To(Equal("tigera-ca-bundle")) clusterRole := rtest.GetResource(resources, enterprise.EsKubeControllerRole, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - Expect(clusterRole.Rules).To(HaveLen(36), "cluster role should have 36 rules") + Expect(clusterRole.Rules).To(HaveLen(26), "cluster role should have 26 rules") Expect(clusterRole.Rules).To(ContainElement( rbacv1.PolicyRule{ APIGroups: []string{""}, @@ -497,57 +323,6 @@ var _ = Describe("kube-controllers rendering tests", func() { })) }) - It("should render all calico-kube-controllers resources for a default configuration using CalicoEnterprise and ClusterType is Management", func() { - expectedResources := []struct { - name string - ns string - group string - version string - kind string - }{ - {name: kubecontrollers.KubeControllerServiceAccount, ns: common.CalicoNamespace, group: "", version: "v1", kind: "ServiceAccount"}, - {name: kubecontrollers.KubeControllerRole, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, - {name: kubecontrollers.KubeControllerRoleBinding, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: kubecontrollers.ManagedClustersWatchRoleBindingName, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: kubecontrollers.KubeController, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "Deployment"}, - {name: applicationlayer.WAFWebhookServiceName, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Service"}, - {name: "tigera-waf.applicationlayer.projectcalico.org", ns: "", group: "admissionregistration.k8s.io", version: "v1", kind: "ValidatingWebhookConfiguration"}, - {name: kubecontrollers.KubeControllerMetrics, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Service"}, - } - - // Override configuration to match expected Enterprise config. - instance.Variant = operatorv1.CalicoEnterprise - cfg.ManagementCluster = &operatorv1.ManagementCluster{} - cfg.MetricsPort = 9094 - // Opt in to the WAF Gateway API add-on so the WAF env vars + RBAC are rendered. - cfg.WAFGatewayExtensionEnabled = true - - component := kubecontrollers.NewCalicoKubeControllers(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - Expect(len(resources)).To(Equal(len(expectedResources))) - - // Should render the correct resources. - i := 0 - for _, expectedRes := range expectedResources { - rtest.ExpectResourceTypeAndObjectMetadata(resources[i], expectedRes.name, expectedRes.ns, expectedRes.group, expectedRes.version, expectedRes.kind) - i++ - } - - // The Deployment should have the correct configuration. - dp := rtest.GetResource(resources, kubecontrollers.KubeController, common.CalicoNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) - - envs := dp.Spec.Template.Spec.Containers[0].Env - Expect(envs).To(ContainElement(corev1.EnvVar{ - Name: "ENABLED_CONTROLLERS", - Value: "node,loadbalancer,service,federatedservices,usage,applicationlayer", - })) - - Expect(len(dp.Spec.Template.Spec.Containers[0].VolumeMounts)).To(Equal(1)) - - Expect(len(dp.Spec.Template.Spec.Volumes)).To(Equal(1)) - Expect(dp.Spec.Template.Spec.Containers[0].Image).To(Equal("test-reg/tigera/calico:" + components.ComponentTigeraCalico.Version)) - }) It("should render all calico-kube-controllers resources for a default configuration using CalicoEnterprise", func() { expectedResources := []struct { name string @@ -614,50 +389,6 @@ var _ = Describe("kube-controllers rendering tests", func() { Expect(dp.Spec.Template.Spec.Containers[0].Image).To(Equal("test-reg/tigera/calico:" + components.ComponentTigeraCalico.Version)) }) - It("should mount the WAF admission webhook serving cert and expose its port when WAF is enabled", func() { - certificateManager, err := certificatemanager.Create(cli, nil, dns.DefaultClusterDomain, common.OperatorNamespace(), certificatemanager.AllowCACreation()) - Expect(err).NotTo(HaveOccurred()) - wafTLS, err := certificateManager.GetOrCreateKeyPair(cli, - applicationlayer.WAFWebhookServerTLSSecretName, - common.OperatorNamespace(), - dns.GetServiceDNSNames(applicationlayer.WAFWebhookServiceName, common.CalicoNamespace, dns.DefaultClusterDomain)) - Expect(err).NotTo(HaveOccurred()) - - instance.Variant = operatorv1.CalicoEnterprise - cfg.WAFGatewayExtensionEnabled = true - cfg.WAFWebhookServerTLS = wafTLS - - component := kubecontrollers.NewCalicoKubeControllers(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - - dp := rtest.GetResource(resources, kubecontrollers.KubeController, common.CalicoNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) - c := dp.Spec.Template.Spec.Containers[0] - - // Serving cert is mounted and advertised to the in-process webhook server. - Expect(dp.Spec.Template.Spec.Volumes).To(ContainElement(wafTLS.Volume())) - Expect(c.VolumeMounts).To(ContainElement(wafTLS.VolumeMount(rmeta.OSTypeLinux))) - Expect(c.Env).To(ContainElement(corev1.EnvVar{ - Name: "WAF_WEBHOOK_CERT_DIR", - Value: filepath.Dir(wafTLS.VolumeMountCertificateFilePath()), - })) - - // In-process webhook port exposed for the tigera-waf-webhook Service. - Expect(c.Ports).To(ContainElement(corev1.ContainerPort{ - Name: "waf-webhook", - ContainerPort: int32(9443), - Protocol: corev1.ProtocolTCP, - })) - - // namespaces patch/update RBAC for the waf-id-range annotation. - clusterRole := rtest.GetResource(resources, "calico-kube-controllers", "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - Expect(clusterRole.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"namespaces"}, - Verbs: []string{"get", "patch", "update"}, - })) - }) - It("should render all es-calico-kube-controllers resources for a default configuration using CalicoEnterprise and ClusterType is Management", func() { expectedResources := []struct { name string @@ -681,8 +412,6 @@ var _ = Describe("kube-controllers rendering tests", func() { cfg.ManagementCluster = &operatorv1.ManagementCluster{} cfg.KubeControllersGatewaySecret = &testutils.KubeControllersUserSecret cfg.MetricsPort = 9094 - // Opt in to the WAF Gateway API add-on so the WAF env vars + RBAC are rendered. - cfg.WAFGatewayExtensionEnabled = true component := enterprise.NewElasticsearchKubeControllers(&cfg) Expect(component.ResolveImages(nil)).To(BeNil()) @@ -716,7 +445,7 @@ var _ = Describe("kube-controllers rendering tests", func() { Expect(dp.Spec.Template.Spec.Containers[0].Image).To(Equal("test-reg/tigera/calico:" + components.ComponentTigeraCalico.Version)) clusterRole := rtest.GetResource(resources, enterprise.EsKubeControllerRole, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - Expect(clusterRole.Rules).To(HaveLen(36), "cluster role should have 36 rules") + Expect(clusterRole.Rules).To(HaveLen(26), "cluster role should have 26 rules") Expect(clusterRole.Rules).To(ContainElement( rbacv1.PolicyRule{ APIGroups: []string{""}, diff --git a/pkg/render/kubecontrollers/waf_pull_secret.go b/pkg/render/kubecontrollers/waf_pull_secret.go deleted file mode 100644 index 02ada09f8a..0000000000 --- a/pkg/render/kubecontrollers/waf_pull_secret.go +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright (c) 2026 Tigera, Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package kubecontrollers - -import ( - "encoding/json" - "fmt" - - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "github.com/tigera/operator/pkg/common" -) - -// MergeWAFPullSecret synthesizes the dedicated WAF wasm pull secret -// (tigera-waf-pull-secret) by merging the registry auths of every Installation -// pull secret. The EnvoyExtensionPolicy image source takes a single -// pullSecretRef, so a merged secret is the only way to honor multiple -// Installation pull secrets for the Coraza wasm OCI pull (e.g. the Tigera pull -// secret plus credentials for a private registry mirror). -// -// If the same registry appears in more than one secret, the first secret in -// Installation order wins. Secrets that cannot be parsed are skipped and their -// names returned, so the caller can log them without failing the reconcile. -// Returns a nil Secret when no registry auths could be collected. -func MergeWAFPullSecret(pullSecrets []*corev1.Secret) (*corev1.Secret, []string) { - merged := map[string]json.RawMessage{} - var skipped []string - for _, s := range pullSecrets { - auths, err := registryAuths(s) - if err != nil { - skipped = append(skipped, s.Name) - continue - } - for registry, auth := range auths { - if _, ok := merged[registry]; !ok { - merged[registry] = auth - } - } - } - if len(merged) == 0 { - return nil, skipped - } - - // Marshalling a map sorts its keys, so the rendered bytes are deterministic - // and do not churn the object on every reconcile. - data, err := json.Marshal(map[string]map[string]json.RawMessage{"auths": merged}) - if err != nil { - // Each auth entry round-trips from a successful Unmarshal above, so - // this cannot fail in practice; treat it as nothing to render. - return nil, skipped - } - - return &corev1.Secret{ - TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, - ObjectMeta: metav1.ObjectMeta{Name: WASMPullSecretName, Namespace: common.CalicoNamespace}, - Type: corev1.SecretTypeDockerConfigJson, - Data: map[string][]byte{corev1.DockerConfigJsonKey: data}, - }, skipped -} - -// registryAuths extracts the per-registry auth entries from a pull secret of -// either the dockerconfigjson type (auths nested under an "auths" key) or the -// legacy dockercfg type (a bare registry -> auth map). -func registryAuths(s *corev1.Secret) (map[string]json.RawMessage, error) { - if raw, ok := s.Data[corev1.DockerConfigJsonKey]; ok { - var cfg struct { - Auths map[string]json.RawMessage `json:"auths"` - } - if err := json.Unmarshal(raw, &cfg); err != nil { - return nil, err - } - if len(cfg.Auths) == 0 { - return nil, fmt.Errorf("secret %s has no auths entries", s.Name) - } - return cfg.Auths, nil - } - if raw, ok := s.Data[corev1.DockerConfigKey]; ok { - var auths map[string]json.RawMessage - if err := json.Unmarshal(raw, &auths); err != nil { - return nil, err - } - if len(auths) == 0 { - return nil, fmt.Errorf("secret %s has no auths entries", s.Name) - } - return auths, nil - } - return nil, fmt.Errorf("secret %s has neither a %s nor a %s key", s.Name, corev1.DockerConfigJsonKey, corev1.DockerConfigKey) -} diff --git a/pkg/render/logstorage/esgateway/esgateway_test.go b/pkg/render/logstorage/esgateway/esgateway_test.go index 7f29d298d7..5a76cc2c34 100644 --- a/pkg/render/logstorage/esgateway/esgateway_test.go +++ b/pkg/render/logstorage/esgateway/esgateway_test.go @@ -37,12 +37,12 @@ import ( "github.com/tigera/operator/pkg/controller/certificatemanager" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/enterprise" "github.com/tigera/operator/pkg/render" relasticsearch "github.com/tigera/operator/pkg/render/common/elasticsearch" rmeta "github.com/tigera/operator/pkg/render/common/meta" "github.com/tigera/operator/pkg/render/common/podaffinity" rtest "github.com/tigera/operator/pkg/render/common/test" - "github.com/tigera/operator/pkg/render/kubecontrollers" "github.com/tigera/operator/pkg/render/testutils" "github.com/tigera/operator/pkg/tls" "github.com/tigera/operator/pkg/tls/certificatemanagement" @@ -81,9 +81,9 @@ var _ = Describe("ES Gateway rendering tests", func() { ESGatewayKeyPair: kp, TrustedBundle: bundle, KubeControllersUserSecrets: []*corev1.Secret{ - {ObjectMeta: metav1.ObjectMeta{Name: kubecontrollers.ElasticsearchKubeControllersUserSecret, Namespace: common.OperatorNamespace()}}, - {ObjectMeta: metav1.ObjectMeta{Name: kubecontrollers.ElasticsearchKubeControllersVerificationUserSecret, Namespace: render.ElasticsearchNamespace}}, - {ObjectMeta: metav1.ObjectMeta{Name: kubecontrollers.ElasticsearchKubeControllersSecureUserSecret, Namespace: render.ElasticsearchNamespace}}, + {ObjectMeta: metav1.ObjectMeta{Name: enterprise.ElasticsearchKubeControllersUserSecret, Namespace: common.OperatorNamespace()}}, + {ObjectMeta: metav1.ObjectMeta{Name: enterprise.ElasticsearchKubeControllersVerificationUserSecret, Namespace: render.ElasticsearchNamespace}}, + {ObjectMeta: metav1.ObjectMeta{Name: enterprise.ElasticsearchKubeControllersSecureUserSecret, Namespace: render.ElasticsearchNamespace}}, }, ClusterDomain: clusterDomain, EsAdminUserName: "elastic", @@ -95,9 +95,9 @@ var _ = Describe("ES Gateway rendering tests", func() { It("should render an ES Gateway deployment and all supporting resources", func() { expectedResources := []client.Object{ &v3.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: PolicyName, Namespace: render.ElasticsearchNamespace}}, - &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: kubecontrollers.ElasticsearchKubeControllersUserSecret, Namespace: common.OperatorNamespace()}}, - &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: kubecontrollers.ElasticsearchKubeControllersVerificationUserSecret, Namespace: render.ElasticsearchNamespace}}, - &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: kubecontrollers.ElasticsearchKubeControllersSecureUserSecret, Namespace: render.ElasticsearchNamespace}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: enterprise.ElasticsearchKubeControllersUserSecret, Namespace: common.OperatorNamespace()}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: enterprise.ElasticsearchKubeControllersVerificationUserSecret, Namespace: render.ElasticsearchNamespace}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: enterprise.ElasticsearchKubeControllersSecureUserSecret, Namespace: render.ElasticsearchNamespace}}, &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: ServiceName, Namespace: render.ElasticsearchNamespace}}, &rbacv1.Role{ObjectMeta: metav1.ObjectMeta{Name: RoleName, Namespace: render.ElasticsearchNamespace}}, &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: RoleName, Namespace: render.ElasticsearchNamespace}}, @@ -134,9 +134,9 @@ var _ = Describe("ES Gateway rendering tests", func() { installation.CertificateManagement = &operatorv1.CertificateManagement{CACert: secret.Data[corev1.TLSCertKey]} expectedResources := []client.Object{ &v3.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: PolicyName, Namespace: render.ElasticsearchNamespace}}, - &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: kubecontrollers.ElasticsearchKubeControllersUserSecret, Namespace: common.OperatorNamespace()}}, - &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: kubecontrollers.ElasticsearchKubeControllersVerificationUserSecret, Namespace: render.ElasticsearchNamespace}}, - &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: kubecontrollers.ElasticsearchKubeControllersSecureUserSecret, Namespace: render.ElasticsearchNamespace}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: enterprise.ElasticsearchKubeControllersUserSecret, Namespace: common.OperatorNamespace()}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: enterprise.ElasticsearchKubeControllersVerificationUserSecret, Namespace: render.ElasticsearchNamespace}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: enterprise.ElasticsearchKubeControllersSecureUserSecret, Namespace: render.ElasticsearchNamespace}}, &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: ServiceName, Namespace: render.ElasticsearchNamespace}}, &rbacv1.Role{ObjectMeta: metav1.ObjectMeta{Name: RoleName, Namespace: render.ElasticsearchNamespace}}, &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: RoleName, Namespace: render.ElasticsearchNamespace}},